2024-11-19 21:54:45,034 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-19 21:54:45,054 main DEBUG Took 0.018278 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 21:54:45,055 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 21:54:45,055 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 21:54:45,056 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 21:54:45,057 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,066 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 21:54:45,082 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,084 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,085 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,087 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,088 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,089 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,091 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,092 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,093 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,093 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,094 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,095 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,095 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,096 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,096 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:54:45,097 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,097 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 21:54:45,099 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:54:45,100 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 21:54:45,102 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 21:54:45,102 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 21:54:45,104 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 21:54:45,105 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 21:54:45,115 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 21:54:45,117 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 21:54:45,119 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 21:54:45,119 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 21:54:45,120 main DEBUG createAppenders(={Console}) 2024-11-19 21:54:45,120 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-19 21:54:45,121 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-19 21:54:45,121 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-19 21:54:45,121 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 21:54:45,122 main DEBUG OutputStream closed 2024-11-19 21:54:45,122 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 21:54:45,122 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 21:54:45,122 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-19 21:54:45,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 21:54:45,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 21:54:45,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 21:54:45,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 21:54:45,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 21:54:45,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 21:54:45,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 21:54:45,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 21:54:45,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 21:54:45,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 21:54:45,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 21:54:45,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 21:54:45,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 21:54:45,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 21:54:45,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 21:54:45,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 21:54:45,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 21:54:45,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 21:54:45,197 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 21:54:45,198 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-19 21:54:45,198 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 21:54:45,199 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-19T21:54:45,210 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-19 21:54:45,212 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 21:54:45,213 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T21:54:45,430 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f 2024-11-19T21:54:45,457 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=394, ProcessCount=11, AvailableMemoryMB=5888 2024-11-19T21:54:45,546 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T21:54:45,634 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:45,648 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/cluster_6ff4e36e-f8a3-37e1-2d59-44f4bfeb3b33, deleteOnExit=true 2024-11-19T21:54:45,781 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/cluster_6ff4e36e-f8a3-37e1-2d59-44f4bfeb3b33/zookeeper_0, clientPort=55554, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/cluster_6ff4e36e-f8a3-37e1-2d59-44f4bfeb3b33/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/cluster_6ff4e36e-f8a3-37e1-2d59-44f4bfeb3b33/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T21:54:45,792 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55554 2024-11-19T21:54:45,793 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:45,880 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab with version=8 2024-11-19T21:54:45,881 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/hbase-staging 2024-11-19T21:54:45,969 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T21:54:46,234 INFO [Time-limited test {}] client.ConnectionUtils(128): master/707262ffa86d:0 server-side Connection retries=45 2024-11-19T21:54:46,243 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:46,244 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:46,255 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T21:54:46,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:46,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T21:54:46,401 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T21:54:46,477 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T21:54:46,512 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T21:54:46,523 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T21:54:46,527 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T21:54:46,558 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 96950 (auto-detected) 2024-11-19T21:54:46,559 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T21:54:46,578 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33907 2024-11-19T21:54:46,603 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33907 connecting to ZooKeeper ensemble=127.0.0.1:55554 2024-11-19T21:54:46,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:339070x0, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T21:54:46,716 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33907-0x10156614e420000 connected 2024-11-19T21:54:46,809 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:46,810 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:46,819 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:46,823 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab, hbase.cluster.distributed=false 2024-11-19T21:54:46,845 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T21:54:46,851 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33907 2024-11-19T21:54:46,851 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33907 2024-11-19T21:54:46,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33907 2024-11-19T21:54:46,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33907 2024-11-19T21:54:46,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33907 2024-11-19T21:54:46,928 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/707262ffa86d:0 server-side Connection retries=45 2024-11-19T21:54:46,929 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:46,929 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:46,930 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T21:54:46,930 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:46,930 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T21:54:46,933 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T21:54:46,936 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T21:54:46,936 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34667 2024-11-19T21:54:46,938 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34667 connecting to ZooKeeper ensemble=127.0.0.1:55554 2024-11-19T21:54:46,939 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:46,940 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:46,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:346670x0, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T21:54:46,974 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34667-0x10156614e420001 connected 2024-11-19T21:54:46,974 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:46,982 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T21:54:46,998 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/bucket.cache 2024-11-19T21:54:47,006 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T21:54:47,008 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T21:54:47,010 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T21:54:47,012 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T21:54:47,012 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T21:54:47,020 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T21:54:47,020 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T21:54:47,020 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/bucket.persistence is complete. 2024-11-19T21:54:47,028 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T21:54:47,032 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T21:54:47,039 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T21:54:47,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34667 2024-11-19T21:54:47,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34667 2024-11-19T21:54:47,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34667 2024-11-19T21:54:47,042 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34667 2024-11-19T21:54:47,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34667 2024-11-19T21:54:47,072 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/707262ffa86d:0 server-side Connection retries=45 2024-11-19T21:54:47,072 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:47,073 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:47,073 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T21:54:47,073 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:47,074 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T21:54:47,074 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T21:54:47,075 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T21:54:47,076 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46439 2024-11-19T21:54:47,079 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46439 connecting to ZooKeeper ensemble=127.0.0.1:55554 2024-11-19T21:54:47,079 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:47,081 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:47,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464390x0, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T21:54:47,099 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:464390x0, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:47,099 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46439-0x10156614e420002 connected 2024-11-19T21:54:47,100 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T21:54:47,101 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/bucket.cache 2024-11-19T21:54:47,102 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T21:54:47,102 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T21:54:47,103 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T21:54:47,107 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T21:54:47,107 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T21:54:47,107 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T21:54:47,108 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T21:54:47,108 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/bucket.persistence is complete. 2024-11-19T21:54:47,111 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T21:54:47,112 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T21:54:47,115 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T21:54:47,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46439 2024-11-19T21:54:47,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46439 2024-11-19T21:54:47,121 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46439 2024-11-19T21:54:47,122 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46439 2024-11-19T21:54:47,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46439 2024-11-19T21:54:47,143 DEBUG [M:0;707262ffa86d:33907 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;707262ffa86d:33907 2024-11-19T21:54:47,144 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/707262ffa86d,33907,1732053286072 2024-11-19T21:54:47,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:47,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:47,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:47,184 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/707262ffa86d,33907,1732053286072 2024-11-19T21:54:47,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T21:54:47,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T21:54:47,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:47,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:47,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:47,216 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T21:54:47,217 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/707262ffa86d,33907,1732053286072 from backup master directory 2024-11-19T21:54:47,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/707262ffa86d,33907,1732053286072 2024-11-19T21:54:47,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:47,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:47,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:47,224 WARN [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T21:54:47,224 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=707262ffa86d,33907,1732053286072 2024-11-19T21:54:47,227 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T21:54:47,229 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T21:54:47,239 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/hbase.id] with ID: 6b92ecbb-028e-497f-b86d-5ec904be3f24 2024-11-19T21:54:47,239 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/.tmp/hbase.id 2024-11-19T21:54:47,245 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/hbase.id] 2024-11-19T21:54:47,278 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:47,283 INFO [master/707262ffa86d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T21:54:47,308 INFO [master/707262ffa86d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-19T21:54:47,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:47,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:47,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:47,365 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:54:47,369 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T21:54:47,419 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:54:47,425 INFO [master/707262ffa86d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:54:47,475 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store 2024-11-19T21:54:47,499 INFO [master/707262ffa86d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T21:54:47,503 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:47,505 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T21:54:47,505 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:47,505 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:47,507 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T21:54:47,507 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:47,508 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:47,508 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732053287504Disabling compacts and flushes for region at 1732053287504Disabling writes for close at 1732053287507 (+3 ms)Writing region close event to WAL at 1732053287508 (+1 ms)Closed at 1732053287508 2024-11-19T21:54:47,509 WARN [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/.initializing 2024-11-19T21:54:47,509 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/WALs/707262ffa86d,33907,1732053286072 2024-11-19T21:54:47,523 INFO [master/707262ffa86d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T21:54:47,537 INFO [master/707262ffa86d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=707262ffa86d%2C33907%2C1732053286072, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/WALs/707262ffa86d,33907,1732053286072, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/oldWALs, maxLogs=10 2024-11-19T21:54:47,579 INFO [master/707262ffa86d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/WALs/707262ffa86d,33907,1732053286072/707262ffa86d%2C33907%2C1732053286072.1732053287543 2024-11-19T21:54:47,580 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T21:54:47,580 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:54:47,581 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:47,584 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,585 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,620 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T21:54:47,655 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:47,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:47,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T21:54:47,662 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:47,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:54:47,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,668 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T21:54:47,668 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:47,668 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:54:47,669 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T21:54:47,673 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:47,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:54:47,674 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,679 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,679 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,683 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,684 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,688 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T21:54:47,691 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:47,698 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:54:47,699 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64615714, jitterRate=-0.037150830030441284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T21:54:47,709 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732053287597Initializing all the Stores at 1732053287598 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053287598Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053287599 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053287599Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053287599Cleaning up temporary data from old regions at 1732053287684 (+85 ms)Region opened successfully at 1732053287709 (+25 ms) 2024-11-19T21:54:47,711 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T21:54:47,751 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dbbf432, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=707262ffa86d/172.17.0.2:0 2024-11-19T21:54:47,793 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T21:54:47,807 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T21:54:47,807 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T21:54:47,811 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T21:54:47,811 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T21:54:47,818 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-19T21:54:47,819 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T21:54:47,850 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T21:54:47,861 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T21:54:47,906 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T21:54:47,909 INFO [master/707262ffa86d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T21:54:47,911 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T21:54:47,922 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T21:54:47,924 INFO [master/707262ffa86d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T21:54:47,928 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T21:54:47,939 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T21:54:47,942 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T21:54:47,947 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T21:54:47,965 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T21:54:47,972 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T21:54:47,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:47,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:47,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:47,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:47,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:47,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:47,992 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=707262ffa86d,33907,1732053286072, sessionid=0x10156614e420000, setting cluster-up flag (Was=false) 2024-11-19T21:54:48,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:48,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:48,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:48,048 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T21:54:48,051 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=707262ffa86d,33907,1732053286072 2024-11-19T21:54:48,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:48,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:48,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:48,097 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T21:54:48,101 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=707262ffa86d,33907,1732053286072 2024-11-19T21:54:48,110 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T21:54:48,129 INFO [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(746): ClusterId : 6b92ecbb-028e-497f-b86d-5ec904be3f24 2024-11-19T21:54:48,129 INFO [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(746): ClusterId : 6b92ecbb-028e-497f-b86d-5ec904be3f24 2024-11-19T21:54:48,131 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T21:54:48,131 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T21:54:48,149 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T21:54:48,149 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T21:54:48,149 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T21:54:48,149 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T21:54:48,157 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T21:54:48,157 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T21:54:48,158 DEBUG [RS:1;707262ffa86d:46439 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f79900d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=707262ffa86d/172.17.0.2:0 2024-11-19T21:54:48,158 DEBUG [RS:0;707262ffa86d:34667 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1632a727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=707262ffa86d/172.17.0.2:0 2024-11-19T21:54:48,172 DEBUG [RS:1;707262ffa86d:46439 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;707262ffa86d:46439 2024-11-19T21:54:48,172 DEBUG [RS:0;707262ffa86d:34667 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;707262ffa86d:34667 2024-11-19T21:54:48,176 INFO [RS:0;707262ffa86d:34667 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T21:54:48,176 INFO [RS:1;707262ffa86d:46439 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T21:54:48,176 INFO [RS:1;707262ffa86d:46439 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T21:54:48,176 INFO [RS:0;707262ffa86d:34667 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T21:54:48,176 DEBUG [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T21:54:48,176 DEBUG [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T21:54:48,184 INFO [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(2659): reportForDuty to master=707262ffa86d,33907,1732053286072 with port=46439, startcode=1732053287071 2024-11-19T21:54:48,184 INFO [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(2659): reportForDuty to master=707262ffa86d,33907,1732053286072 with port=34667, startcode=1732053286895 2024-11-19T21:54:48,189 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T21:54:48,197 DEBUG [RS:0;707262ffa86d:34667 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T21:54:48,197 DEBUG [RS:1;707262ffa86d:46439 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T21:54:48,203 INFO [master/707262ffa86d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T21:54:48,211 INFO [master/707262ffa86d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T21:54:48,218 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 707262ffa86d,33907,1732053286072 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T21:54:48,226 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:54:48,226 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:54:48,227 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:54:48,227 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:54:48,227 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/707262ffa86d:0, corePoolSize=10, maxPoolSize=10 2024-11-19T21:54:48,227 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,228 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/707262ffa86d:0, corePoolSize=2, maxPoolSize=2 2024-11-19T21:54:48,228 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,233 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T21:54:48,233 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732053318233 2024-11-19T21:54:48,234 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T21:54:48,235 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T21:54:48,236 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T21:54:48,239 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T21:54:48,241 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T21:54:48,243 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T21:54:48,243 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:48,243 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T21:54:48,243 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T21:54:48,255 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,263 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T21:54:48,265 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T21:54:48,265 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T21:54:48,269 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T21:54:48,270 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T21:54:48,271 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35847, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T21:54:48,271 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40937, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T21:54:48,273 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.large.0-1732053288271,5,FailOnTimeoutGroup] 2024-11-19T21:54:48,274 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.small.0-1732053288273,5,FailOnTimeoutGroup] 2024-11-19T21:54:48,274 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,274 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T21:54:48,275 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,276 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,277 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T21:54:48,278 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33907 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 707262ffa86d,34667,1732053286895 2024-11-19T21:54:48,278 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab 2024-11-19T21:54:48,280 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33907 {}] master.ServerManager(517): Registering regionserver=707262ffa86d,34667,1732053286895 2024-11-19T21:54:48,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:48,291 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T21:54:48,291 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33907 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 707262ffa86d,46439,1732053287071 2024-11-19T21:54:48,291 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33907 {}] master.ServerManager(517): Registering regionserver=707262ffa86d,46439,1732053287071 2024-11-19T21:54:48,295 DEBUG [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab 2024-11-19T21:54:48,295 DEBUG [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab 2024-11-19T21:54:48,295 DEBUG [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T21:54:48,295 DEBUG [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T21:54:48,295 DEBUG [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T21:54:48,295 DEBUG [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T21:54:48,297 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T21:54:48,297 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:48,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:48,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T21:54:48,301 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T21:54:48,301 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:48,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:48,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T21:54:48,305 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T21:54:48,305 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:48,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:48,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T21:54:48,308 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T21:54:48,308 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:48,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:48,309 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T21:54:48,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740 2024-11-19T21:54:48,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740 2024-11-19T21:54:48,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T21:54:48,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T21:54:48,310 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T21:54:48,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T21:54:48,316 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:54:48,317 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70224628, jitterRate=0.04642850160598755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T21:54:48,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732053288288Initializing all the Stores at 1732053288288Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053288288Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053288291 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053288291Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053288291Cleaning up temporary data from old regions at 1732053288310 (+19 ms)Region opened successfully at 1732053288321 (+11 ms) 2024-11-19T21:54:48,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T21:54:48,321 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T21:54:48,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T21:54:48,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T21:54:48,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T21:54:48,323 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T21:54:48,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732053288321Disabling compacts and flushes for region at 1732053288321Disabling writes for close at 1732053288322 (+1 ms)Writing region close event to WAL at 1732053288323 (+1 ms)Closed at 1732053288323 2024-11-19T21:54:48,324 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T21:54:48,324 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T21:54:48,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T21:54:48,337 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T21:54:48,339 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T21:54:48,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T21:54:48,344 DEBUG [RS:1;707262ffa86d:46439 {}] zookeeper.ZKUtil(111): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/707262ffa86d,46439,1732053287071 2024-11-19T21:54:48,344 DEBUG [RS:0;707262ffa86d:34667 {}] zookeeper.ZKUtil(111): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/707262ffa86d,34667,1732053286895 2024-11-19T21:54:48,344 WARN [RS:0;707262ffa86d:34667 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T21:54:48,344 WARN [RS:1;707262ffa86d:46439 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T21:54:48,344 INFO [RS:0;707262ffa86d:34667 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:54:48,344 INFO [RS:1;707262ffa86d:46439 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:54:48,344 DEBUG [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,34667,1732053286895 2024-11-19T21:54:48,344 DEBUG [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,46439,1732053287071 2024-11-19T21:54:48,346 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [707262ffa86d,46439,1732053287071] 2024-11-19T21:54:48,346 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [707262ffa86d,34667,1732053286895] 2024-11-19T21:54:48,372 INFO [RS:0;707262ffa86d:34667 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T21:54:48,372 INFO [RS:1;707262ffa86d:46439 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T21:54:48,391 INFO [RS:0;707262ffa86d:34667 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T21:54:48,391 INFO [RS:1;707262ffa86d:46439 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T21:54:48,396 INFO [RS:0;707262ffa86d:34667 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T21:54:48,396 INFO [RS:1;707262ffa86d:46439 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T21:54:48,396 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,396 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,397 INFO [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T21:54:48,397 INFO [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T21:54:48,402 INFO [RS:0;707262ffa86d:34667 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T21:54:48,402 INFO [RS:1;707262ffa86d:46439 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T21:54:48,403 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,403 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,404 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,404 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,404 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,404 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,404 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,404 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,404 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,404 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,404 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,404 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/707262ffa86d:0, corePoolSize=2, maxPoolSize=2 2024-11-19T21:54:48,404 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,404 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,404 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/707262ffa86d:0, corePoolSize=2, maxPoolSize=2 2024-11-19T21:54:48,404 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,405 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,405 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,405 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,405 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,405 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,405 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,405 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,405 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:54:48,405 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,405 DEBUG [RS:0;707262ffa86d:34667 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:54:48,405 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,406 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:48,406 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:54:48,406 DEBUG [RS:1;707262ffa86d:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:54:48,406 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,407 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,407 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,407 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,407 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,407 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,34667,1732053286895-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T21:54:48,411 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,411 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,411 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,411 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,411 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,412 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,46439,1732053287071-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T21:54:48,429 INFO [RS:1;707262ffa86d:46439 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T21:54:48,431 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,46439,1732053287071-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,431 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,431 INFO [RS:1;707262ffa86d:46439 {}] regionserver.Replication(171): 707262ffa86d,46439,1732053287071 started 2024-11-19T21:54:48,433 INFO [RS:0;707262ffa86d:34667 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T21:54:48,433 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,34667,1732053286895-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,433 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,434 INFO [RS:0;707262ffa86d:34667 {}] regionserver.Replication(171): 707262ffa86d,34667,1732053286895 started 2024-11-19T21:54:48,448 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,449 INFO [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(1482): Serving as 707262ffa86d,46439,1732053287071, RpcServer on 707262ffa86d/172.17.0.2:46439, sessionid=0x10156614e420002 2024-11-19T21:54:48,450 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T21:54:48,450 DEBUG [RS:1;707262ffa86d:46439 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 707262ffa86d,46439,1732053287071 2024-11-19T21:54:48,450 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,46439,1732053287071' 2024-11-19T21:54:48,450 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T21:54:48,451 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T21:54:48,452 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T21:54:48,452 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T21:54:48,452 DEBUG [RS:1;707262ffa86d:46439 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 707262ffa86d,46439,1732053287071 2024-11-19T21:54:48,452 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,46439,1732053287071' 2024-11-19T21:54:48,452 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T21:54:48,453 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T21:54:48,454 DEBUG [RS:1;707262ffa86d:46439 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T21:54:48,454 INFO [RS:1;707262ffa86d:46439 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T21:54:48,454 INFO [RS:1;707262ffa86d:46439 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T21:54:48,457 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:48,457 INFO [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(1482): Serving as 707262ffa86d,34667,1732053286895, RpcServer on 707262ffa86d/172.17.0.2:34667, sessionid=0x10156614e420001 2024-11-19T21:54:48,457 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T21:54:48,457 DEBUG [RS:0;707262ffa86d:34667 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 707262ffa86d,34667,1732053286895 2024-11-19T21:54:48,457 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,34667,1732053286895' 2024-11-19T21:54:48,457 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T21:54:48,458 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T21:54:48,459 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T21:54:48,459 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T21:54:48,459 DEBUG [RS:0;707262ffa86d:34667 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 707262ffa86d,34667,1732053286895 2024-11-19T21:54:48,459 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,34667,1732053286895' 2024-11-19T21:54:48,459 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T21:54:48,460 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T21:54:48,461 DEBUG [RS:0;707262ffa86d:34667 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T21:54:48,461 INFO [RS:0;707262ffa86d:34667 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T21:54:48,461 INFO [RS:0;707262ffa86d:34667 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T21:54:48,490 WARN [707262ffa86d:33907 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-19T21:54:48,560 INFO [RS:1;707262ffa86d:46439 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T21:54:48,562 INFO [RS:1;707262ffa86d:46439 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=707262ffa86d%2C46439%2C1732053287071, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,46439,1732053287071, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs, maxLogs=55 2024-11-19T21:54:48,562 INFO [RS:0;707262ffa86d:34667 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T21:54:48,563 INFO [RS:0;707262ffa86d:34667 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=707262ffa86d%2C34667%2C1732053286895, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,34667,1732053286895, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs, maxLogs=55 2024-11-19T21:54:48,587 INFO [RS:1;707262ffa86d:46439 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,46439,1732053287071/707262ffa86d%2C46439%2C1732053287071.1732053288566 2024-11-19T21:54:48,588 DEBUG [RS:1;707262ffa86d:46439 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T21:54:48,588 INFO [RS:0;707262ffa86d:34667 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,34667,1732053286895/707262ffa86d%2C34667%2C1732053286895.1732053288566 2024-11-19T21:54:48,588 DEBUG [RS:0;707262ffa86d:34667 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T21:54:48,746 DEBUG [707262ffa86d:33907 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-19T21:54:48,760 DEBUG [707262ffa86d:33907 {}] balancer.BalancerClusterState(204): Hosts are {707262ffa86d=0} racks are {/default-rack=0} 2024-11-19T21:54:48,764 DEBUG [707262ffa86d:33907 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T21:54:48,764 DEBUG [707262ffa86d:33907 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T21:54:48,764 DEBUG [707262ffa86d:33907 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T21:54:48,764 DEBUG [707262ffa86d:33907 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T21:54:48,765 INFO [707262ffa86d:33907 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T21:54:48,765 INFO [707262ffa86d:33907 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T21:54:48,765 DEBUG [707262ffa86d:33907 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T21:54:48,771 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=707262ffa86d,34667,1732053286895 2024-11-19T21:54:48,776 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 707262ffa86d,34667,1732053286895, state=OPENING 2024-11-19T21:54:48,806 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T21:54:48,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:48,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:48,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:48,815 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:48,815 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:48,815 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:48,817 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T21:54:48,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=707262ffa86d,34667,1732053286895}] 2024-11-19T21:54:48,995 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T21:54:48,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53425, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T21:54:49,006 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T21:54:49,006 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:54:49,008 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T21:54:49,009 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=707262ffa86d%2C34667%2C1732053286895.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,34667,1732053286895, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs, maxLogs=55 2024-11-19T21:54:49,029 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,34667,1732053286895/707262ffa86d%2C34667%2C1732053286895.meta.1732053289012.meta 2024-11-19T21:54:49,029 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T21:54:49,029 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:54:49,031 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T21:54:49,033 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T21:54:49,037 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T21:54:49,040 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T21:54:49,041 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:49,041 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T21:54:49,041 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T21:54:49,043 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T21:54:49,044 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T21:54:49,044 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:49,044 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:49,044 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T21:54:49,045 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T21:54:49,045 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:49,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:49,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T21:54:49,046 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T21:54:49,046 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:49,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:49,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T21:54:49,047 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T21:54:49,048 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:49,048 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:49,048 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T21:54:49,049 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740 2024-11-19T21:54:49,050 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740 2024-11-19T21:54:49,050 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T21:54:49,050 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T21:54:49,050 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T21:54:49,051 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T21:54:49,052 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60491914, jitterRate=-0.09860023856163025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T21:54:49,052 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T21:54:49,054 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732053289042Writing region info on filesystem at 1732053289042Initializing all the Stores at 1732053289042Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053289042Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053289043 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053289043Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053289043Cleaning up temporary data from old regions at 1732053289050 (+7 ms)Running coprocessor post-open hooks at 1732053289052 (+2 ms)Region opened successfully at 1732053289054 (+2 ms) 2024-11-19T21:54:49,060 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732053288984 2024-11-19T21:54:49,070 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T21:54:49,070 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T21:54:49,071 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=707262ffa86d,34667,1732053286895 2024-11-19T21:54:49,072 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 707262ffa86d,34667,1732053286895, state=OPEN 2024-11-19T21:54:49,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T21:54:49,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T21:54:49,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T21:54:49,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:49,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:49,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:49,090 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=707262ffa86d,34667,1732053286895 2024-11-19T21:54:49,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T21:54:49,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=707262ffa86d,34667,1732053286895 in 272 msec 2024-11-19T21:54:49,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T21:54:49,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 767 msec 2024-11-19T21:54:49,100 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T21:54:49,101 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T21:54:49,119 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:54:49,120 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,34667,1732053286895, seqNum=-1] 2024-11-19T21:54:49,138 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:54:49,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48527, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:54:49,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0180 sec 2024-11-19T21:54:49,157 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732053289157, completionTime=-1 2024-11-19T21:54:49,160 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-19T21:54:49,161 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-19T21:54:49,190 INFO [master/707262ffa86d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-19T21:54:49,190 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732053349190 2024-11-19T21:54:49,191 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732053409190 2024-11-19T21:54:49,191 INFO [master/707262ffa86d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-19T21:54:49,193 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-19T21:54:49,199 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,33907,1732053286072-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:49,200 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,33907,1732053286072-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:49,200 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,33907,1732053286072-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:49,201 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-707262ffa86d:33907, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:49,202 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:49,202 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:49,208 DEBUG [master/707262ffa86d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T21:54:49,233 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.009sec 2024-11-19T21:54:49,234 INFO [master/707262ffa86d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T21:54:49,236 INFO [master/707262ffa86d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T21:54:49,237 INFO [master/707262ffa86d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T21:54:49,237 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T21:54:49,237 INFO [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T21:54:49,238 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,33907,1732053286072-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T21:54:49,239 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,33907,1732053286072-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T21:54:49,240 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T21:54:49,241 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T21:54:49,241 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,33907,1732053286072-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:49,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:54:49,355 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T21:54:49,355 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T21:54:49,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,33907,-1 for getting cluster id 2024-11-19T21:54:49,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:54:49,374 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b92ecbb-028e-497f-b86d-5ec904be3f24' 2024-11-19T21:54:49,376 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:54:49,376 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b92ecbb-028e-497f-b86d-5ec904be3f24" 2024-11-19T21:54:49,377 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ed9e186, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:54:49,377 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,33907,-1] 2024-11-19T21:54:49,379 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:54:49,381 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:49,381 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:54:49,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:54:49,384 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:54:49,390 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,34667,1732053286895, seqNum=-1] 2024-11-19T21:54:49,390 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:54:49,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:54:49,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=707262ffa86d,33907,1732053286072 2024-11-19T21:54:49,416 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T21:54:49,420 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 707262ffa86d,33907,1732053286072 2024-11-19T21:54:49,422 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5a7eb225 2024-11-19T21:54:49,423 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T21:54:49,426 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T21:54:49,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732053289413', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:54:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732053289413 2024-11-19T21:54:49,438 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732053289413 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T21:54:49,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732053289413" procId is: 4 2024-11-19T21:54:49,440 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:49,442 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732053289413 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T21:54:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:54:49,454 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732053289413-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 42590388ea221dcf7856712664520cbd, NAME => 'testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732053289413', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab 2024-11-19T21:54:49,459 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732053289413-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:49,460 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732053289413-pool-0 {}] regionserver.HRegion(1722): Closing 42590388ea221dcf7856712664520cbd, disabling compactions & flushes 2024-11-19T21:54:49,460 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732053289413-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:49,460 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732053289413-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:49,460 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732053289413-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. after waiting 0 ms 2024-11-19T21:54:49,460 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732053289413-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:49,460 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732053289413-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:49,460 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732053289413-pool-0 {}] regionserver.HRegion(1676): Region close journal for 42590388ea221dcf7856712664520cbd: Waiting for close lock at 1732053289460Disabling compacts and flushes for region at 1732053289460Disabling writes for close at 1732053289460Writing region close event to WAL at 1732053289460Closed at 1732053289460 2024-11-19T21:54:49,461 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732053289413 execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T21:54:49,466 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732053289461"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732053289461"}]},"ts":"1732053289461"} 2024-11-19T21:54:49,470 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T21:54:49,471 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732053289413 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T21:54:49,474 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732053289413","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732053289472"}]},"ts":"1732053289472"} 2024-11-19T21:54:49,478 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732053289413, state=ENABLING in hbase:meta 2024-11-19T21:54:49,479 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {707262ffa86d=0} racks are {/default-rack=0} 2024-11-19T21:54:49,479 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T21:54:49,479 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T21:54:49,479 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T21:54:49,479 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T21:54:49,479 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T21:54:49,479 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T21:54:49,479 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T21:54:49,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732053289413, region=42590388ea221dcf7856712664520cbd, ASSIGN}] 2024-11-19T21:54:49,482 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732053289413, region=42590388ea221dcf7856712664520cbd, ASSIGN 2024-11-19T21:54:49,483 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732053289413, region=42590388ea221dcf7856712664520cbd, ASSIGN; state=OFFLINE, location=707262ffa86d,34667,1732053286895; forceNewPlan=false, retain=false 2024-11-19T21:54:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:54:49,640 INFO [707262ffa86d:33907 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T21:54:49,642 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=42590388ea221dcf7856712664520cbd, regionState=OPENING, regionLocation=707262ffa86d,34667,1732053286895 2024-11-19T21:54:49,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732053289413, region=42590388ea221dcf7856712664520cbd, ASSIGN because future has completed 2024-11-19T21:54:49,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42590388ea221dcf7856712664520cbd, server=707262ffa86d,34667,1732053286895}] 2024-11-19T21:54:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:54:49,806 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:49,806 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 42590388ea221dcf7856712664520cbd, NAME => 'testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:54:49,807 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732053289413 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:49,807 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:49,807 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:49,807 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:49,808 INFO [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:49,812 INFO [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42590388ea221dcf7856712664520cbd columnFamilyName family 2024-11-19T21:54:49,812 DEBUG [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:49,813 INFO [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] regionserver.HStore(327): Store=42590388ea221dcf7856712664520cbd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:54:49,813 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:49,814 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd 2024-11-19T21:54:49,814 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd 2024-11-19T21:54:49,814 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:49,814 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:49,815 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:49,820 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:54:49,821 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 42590388ea221dcf7856712664520cbd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59013112, jitterRate=-0.12063610553741455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T21:54:49,821 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:49,822 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 42590388ea221dcf7856712664520cbd: Running coprocessor pre-open hook at 1732053289807Writing region info on filesystem at 1732053289807Initializing all the Stores at 1732053289808 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053289808Cleaning up temporary data from old regions at 1732053289814 (+6 ms)Running coprocessor post-open hooks at 1732053289821 (+7 ms)Region opened successfully at 1732053289822 (+1 ms) 2024-11-19T21:54:49,823 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd., pid=6, masterSystemTime=1732053289803 2024-11-19T21:54:49,826 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:49,826 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:49,827 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=42590388ea221dcf7856712664520cbd, regionState=OPEN, openSeqNum=2, regionLocation=707262ffa86d,34667,1732053286895 2024-11-19T21:54:49,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42590388ea221dcf7856712664520cbd, server=707262ffa86d,34667,1732053286895 because future has completed 2024-11-19T21:54:49,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T21:54:49,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 42590388ea221dcf7856712664520cbd, server=707262ffa86d,34667,1732053286895 in 183 msec 2024-11-19T21:54:49,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T21:54:49,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732053289413, region=42590388ea221dcf7856712664520cbd, ASSIGN in 353 msec 2024-11-19T21:54:49,837 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732053289413 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T21:54:49,837 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732053289413","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732053289837"}]},"ts":"1732053289837"} 2024-11-19T21:54:49,840 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732053289413, state=ENABLED in hbase:meta 2024-11-19T21:54:49,841 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732053289413 execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T21:54:49,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732053289413 in 408 msec 2024-11-19T21:54:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:54:50,080 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732053289413 get assigned. Timeout = 60000ms 2024-11-19T21:54:50,080 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732053289413 completed 2024-11-19T21:54:50,082 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T21:54:50,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732053289413 assigned to meta. Checking AM states. 2024-11-19T21:54:50,087 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T21:54:50,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732053289413 assigned. 2024-11-19T21:54:50,096 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732053289413', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd., hostname=707262ffa86d,34667,1732053286895, seqNum=2] 2024-11-19T21:54:50,108 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 42590388ea221dcf7856712664520cbd 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-19T21:54:50,134 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b469939c07f64a30af910994e0f11f03 2024-11-19T21:54:50,174 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/.tmp/family/b469939c07f64a30af910994e0f11f03 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-19T21:54:50,180 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/.tmp/family/b469939c07f64a30af910994e0f11f03 2024-11-19T21:54:50,222 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/.tmp/family/b469939c07f64a30af910994e0f11f03 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03 2024-11-19T21:54:50,224 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03, delay=949 ms 2024-11-19T21:54:50,226 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03, entries=2, sequenceid=6, filesize=4.9 K 2024-11-19T21:54:50,232 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 42590388ea221dcf7856712664520cbd in 124ms, sequenceid=6, compaction requested=false 2024-11-19T21:54:50,232 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732053289413' 2024-11-19T21:54:50,234 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 42590388ea221dcf7856712664520cbd: 2024-11-19T21:54:51,177 DEBUG [hfile-prefetch-1732053290224 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b469939c07f64a30af910994e0f11f03_0. 2024-11-19T21:54:51,183 DEBUG [hfile-prefetch-1732053290224 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b469939c07f64a30af910994e0f11f03_117. 2024-11-19T21:54:51,184 DEBUG [hfile-prefetch-1732053290224 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03, with total blocks 2, and data blocks 1 2024-11-19T21:54:51,185 DEBUG [hfile-prefetch-1732053290224 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-19T21:54:51,185 DEBUG [hfile-prefetch-1732053290224 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-19T21:54:51,185 DEBUG [hfile-prefetch-1732053290224 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03. Will sleep 100ms and try the verification again. 2024-11-19T21:54:51,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=42590388ea221dcf7856712664520cbd, source=707262ffa86d,34667,1732053286895, destination=707262ffa86d,46439,1732053287071, warming up region on 707262ffa86d,46439,1732053287071 2024-11-19T21:54:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T21:54:51,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=42590388ea221dcf7856712664520cbd, source=707262ffa86d,34667,1732053286895, destination=707262ffa86d,46439,1732053287071, running balancer 2024-11-19T21:54:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732053289413, region=42590388ea221dcf7856712664520cbd, REOPEN/MOVE 2024-11-19T21:54:51,256 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732053289413, region=42590388ea221dcf7856712664520cbd, REOPEN/MOVE 2024-11-19T21:54:51,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57483, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T21:54:51,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46439 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46439 {}] regionserver.HRegion(7855): Warmup {ENCODED => 42590388ea221dcf7856712664520cbd, NAME => 'testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:54:51,258 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=42590388ea221dcf7856712664520cbd, regionState=CLOSING, regionLocation=707262ffa86d,34667,1732053286895 2024-11-19T21:54:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46439 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:51,259 INFO [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,260 INFO [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42590388ea221dcf7856712664520cbd columnFamilyName family 2024-11-19T21:54:51,260 DEBUG [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:51,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732053289413, region=42590388ea221dcf7856712664520cbd, REOPEN/MOVE because future has completed 2024-11-19T21:54:51,262 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-19T21:54:51,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42590388ea221dcf7856712664520cbd, server=707262ffa86d,34667,1732053286895}] 2024-11-19T21:54:51,263 DEBUG [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03 2024-11-19T21:54:51,263 INFO [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] regionserver.HStore(327): Store=42590388ea221dcf7856712664520cbd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:54:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46439 {}] regionserver.HRegion(1722): Closing 42590388ea221dcf7856712664520cbd, disabling compactions & flushes 2024-11-19T21:54:51,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46439 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46439 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46439 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. after waiting 0 ms 2024-11-19T21:54:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46439 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,265 DEBUG [StoreFileCloser-42590388ea221dcf7856712664520cbd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03 2024-11-19T21:54:51,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46439 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46439 {}] regionserver.HRegion(1676): Region close journal for 42590388ea221dcf7856712664520cbd: Waiting for close lock at 1732053291264Disabling compacts and flushes for region at 1732053291264Disabling writes for close at 1732053291264Writing region close event to WAL at 1732053291266 (+2 ms)Closed at 1732053291266 2024-11-19T21:54:51,267 WARN [hfile-prefetch-1732053290224 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-19T21:54:51,268 DEBUG [hfile-prefetch-1732053290224 {}] hfile.PrefetchExecutor(121): Prefetch completed for b469939c07f64a30af910994e0f11f03 2024-11-19T21:54:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33907 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-19T21:54:51,310 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T21:54:51,310 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T21:54:51,310 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T21:54:51,310 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T21:54:51,314 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T21:54:51,314 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T21:54:51,315 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T21:54:51,315 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T21:54:51,329 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T21:54:51,329 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732053291310 2024-11-19T21:54:51,329 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T21:54:51,331 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-19T21:54:51,331 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732053291314 2024-11-19T21:54:51,331 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T21:54:51,423 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,423 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-19T21:54:51,424 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 42590388ea221dcf7856712664520cbd, disabling compactions & flushes 2024-11-19T21:54:51,424 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,424 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,424 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. after waiting 0 ms 2024-11-19T21:54:51,424 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,427 DEBUG [StoreFileCloser-42590388ea221dcf7856712664520cbd-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file b469939c07f64a30af910994e0f11f03, starting offset: 0, end offset: 9223372036854775807 2024-11-19T21:54:51,429 DEBUG [StoreFileCloser-42590388ea221dcf7856712664520cbd-family-1 {}] bucket.BucketCache(856): removed key b469939c07f64a30af910994e0f11f03_0 from back map with offset lock 2094080 in the evict process 2024-11-19T21:54:51,432 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T21:54:51,432 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T21:54:51,432 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T21:54:51,432 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T21:54:51,435 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-19T21:54:51,436 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T21:54:51,436 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732053291432 2024-11-19T21:54:51,436 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T21:54:51,437 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,437 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 42590388ea221dcf7856712664520cbd: Waiting for close lock at 1732053291424Running coprocessor pre-close hooks at 1732053291424Disabling compacts and flushes for region at 1732053291424Disabling writes for close at 1732053291424Writing region close event to WAL at 1732053291430 (+6 ms)Running coprocessor post-close hooks at 1732053291436 (+6 ms)Closed at 1732053291437 (+1 ms) 2024-11-19T21:54:51,437 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 42590388ea221dcf7856712664520cbd move to 707262ffa86d,46439,1732053287071 record at close sequenceid=6 2024-11-19T21:54:51,441 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,441 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=42590388ea221dcf7856712664520cbd, regionState=CLOSED 2024-11-19T21:54:51,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42590388ea221dcf7856712664520cbd, server=707262ffa86d,34667,1732053286895 because future has completed 2024-11-19T21:54:51,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T21:54:51,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 42590388ea221dcf7856712664520cbd, server=707262ffa86d,34667,1732053286895 in 182 msec 2024-11-19T21:54:51,449 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732053289413, region=42590388ea221dcf7856712664520cbd, REOPEN/MOVE; state=CLOSED, location=707262ffa86d,46439,1732053287071; forceNewPlan=false, retain=false 2024-11-19T21:54:51,600 INFO [707262ffa86d:33907 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T21:54:51,601 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=42590388ea221dcf7856712664520cbd, regionState=OPENING, regionLocation=707262ffa86d,46439,1732053287071 2024-11-19T21:54:51,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732053289413, region=42590388ea221dcf7856712664520cbd, REOPEN/MOVE because future has completed 2024-11-19T21:54:51,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42590388ea221dcf7856712664520cbd, server=707262ffa86d,46439,1732053287071}] 2024-11-19T21:54:51,767 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,767 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 42590388ea221dcf7856712664520cbd, NAME => 'testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:54:51,768 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732053289413 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,769 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:51,769 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,769 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,771 INFO [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,772 INFO [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42590388ea221dcf7856712664520cbd columnFamilyName family 2024-11-19T21:54:51,772 DEBUG [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:51,774 DEBUG [StoreFileOpener-42590388ea221dcf7856712664520cbd-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03, delay=926 ms 2024-11-19T21:54:51,775 DEBUG [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03 2024-11-19T21:54:51,775 INFO [StoreOpener-42590388ea221dcf7856712664520cbd-1 {}] regionserver.HStore(327): Store=42590388ea221dcf7856712664520cbd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:54:51,776 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,777 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,777 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,777 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,777 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,782 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,783 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 42590388ea221dcf7856712664520cbd; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74111253, jitterRate=0.1043437272310257}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T21:54:51,783 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,785 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 42590388ea221dcf7856712664520cbd: Running coprocessor pre-open hook at 1732053291770Writing region info on filesystem at 1732053291770Initializing all the Stores at 1732053291770Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053291770Cleaning up temporary data from old regions at 1732053291777 (+7 ms)Running coprocessor post-open hooks at 1732053291783 (+6 ms)Region opened successfully at 1732053291784 (+1 ms) 2024-11-19T21:54:51,786 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd., pid=9, masterSystemTime=1732053291762 2024-11-19T21:54:51,789 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,789 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,789 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=42590388ea221dcf7856712664520cbd, regionState=OPEN, openSeqNum=10, regionLocation=707262ffa86d,46439,1732053287071 2024-11-19T21:54:51,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42590388ea221dcf7856712664520cbd, server=707262ffa86d,46439,1732053287071 because future has completed 2024-11-19T21:54:51,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-19T21:54:51,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 42590388ea221dcf7856712664520cbd, server=707262ffa86d,46439,1732053287071 in 186 msec 2024-11-19T21:54:51,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732053289413, region=42590388ea221dcf7856712664520cbd, REOPEN/MOVE in 540 msec 2024-11-19T21:54:51,809 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T21:54:51,809 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:54:51,809 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:54:51,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:51,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:51,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:54:51,815 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T21:54:51,815 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2143720667, stopped=false 2024-11-19T21:54:51,815 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=707262ffa86d,33907,1732053286072 2024-11-19T21:54:51,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:51,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:51,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:51,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:51,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:51,872 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T21:54:51,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:51,873 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:54:51,873 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:54:51,873 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:51,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:51,874 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:51,874 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:51,875 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '707262ffa86d,34667,1732053286895' ***** 2024-11-19T21:54:51,875 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T21:54:51,875 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '707262ffa86d,46439,1732053287071' ***** 2024-11-19T21:54:51,875 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T21:54:51,875 INFO [RS:0;707262ffa86d:34667 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T21:54:51,875 INFO [RS:1;707262ffa86d:46439 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T21:54:51,876 INFO [RS:0;707262ffa86d:34667 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/bucket.persistence 2024-11-19T21:54:51,876 INFO [RS:1;707262ffa86d:46439 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/bucket.persistence 2024-11-19T21:54:51,876 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T21:54:51,876 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T21:54:51,876 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T21:54:51,876 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T21:54:51,876 INFO [RS:0;707262ffa86d:34667 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T21:54:51,876 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T21:54:51,876 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T21:54:51,877 INFO [RS:1;707262ffa86d:46439 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T21:54:51,876 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T21:54:51,877 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T21:54:51,886 DEBUG [RS:0;707262ffa86d:34667 {}] bucket.BucketCache(1487): Thread RS:0;707262ffa86d:34667 started persisting bucket cache to file 2024-11-19T21:54:51,887 DEBUG [RS:0;707262ffa86d:34667 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T21:54:51,887 DEBUG [RS:0;707262ffa86d:34667 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T21:54:51,887 DEBUG [RS:1;707262ffa86d:46439 {}] bucket.BucketCache(1487): Thread RS:1;707262ffa86d:46439 started persisting bucket cache to file 2024-11-19T21:54:51,887 DEBUG [RS:1;707262ffa86d:46439 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T21:54:51,887 DEBUG [RS:1;707262ffa86d:46439 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T21:54:51,889 DEBUG [RS:1;707262ffa86d:46439 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T21:54:51,889 DEBUG [RS:0;707262ffa86d:34667 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T21:54:51,890 DEBUG [RS:1;707262ffa86d:46439 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732053291887 2024-11-19T21:54:51,890 DEBUG [RS:0;707262ffa86d:34667 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732053291886 2024-11-19T21:54:51,890 DEBUG [RS:1;707262ffa86d:46439 {}] bucket.BucketCache(1509): Thread RS:1;707262ffa86d:46439 finished persisting bucket cache to file, renaming 2024-11-19T21:54:51,890 DEBUG [RS:0;707262ffa86d:34667 {}] bucket.BucketCache(1509): Thread RS:0;707262ffa86d:34667 finished persisting bucket cache to file, renaming 2024-11-19T21:54:51,890 INFO [RS:1;707262ffa86d:46439 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T21:54:51,890 INFO [RS:0;707262ffa86d:34667 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T21:54:51,890 INFO [RS:1;707262ffa86d:46439 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T21:54:51,890 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T21:54:51,890 INFO [RS:0;707262ffa86d:34667 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T21:54:51,890 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T21:54:51,890 INFO [RS:0;707262ffa86d:34667 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T21:54:51,890 INFO [RS:1;707262ffa86d:46439 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T21:54:51,890 INFO [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(959): stopping server 707262ffa86d,34667,1732053286895 2024-11-19T21:54:51,890 INFO [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(3091): Received CLOSE for 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,890 INFO [RS:0;707262ffa86d:34667 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T21:54:51,891 INFO [RS:0;707262ffa86d:34667 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;707262ffa86d:34667. 2024-11-19T21:54:51,891 DEBUG [RS:0;707262ffa86d:34667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:54:51,891 DEBUG [RS:0;707262ffa86d:34667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:51,891 INFO [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(959): stopping server 707262ffa86d,46439,1732053287071 2024-11-19T21:54:51,891 INFO [RS:1;707262ffa86d:46439 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T21:54:51,891 INFO [RS:0;707262ffa86d:34667 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T21:54:51,891 INFO [RS:0;707262ffa86d:34667 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T21:54:51,891 INFO [RS:1;707262ffa86d:46439 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;707262ffa86d:46439. 2024-11-19T21:54:51,891 INFO [RS:0;707262ffa86d:34667 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T21:54:51,891 INFO [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T21:54:51,891 DEBUG [RS:1;707262ffa86d:46439 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:54:51,891 DEBUG [RS:1;707262ffa86d:46439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:51,891 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T21:54:51,891 INFO [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T21:54:51,891 INFO [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T21:54:51,892 DEBUG [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(1325): Online Regions={42590388ea221dcf7856712664520cbd=testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd.} 2024-11-19T21:54:51,892 DEBUG [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T21:54:51,892 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 42590388ea221dcf7856712664520cbd, disabling compactions & flushes 2024-11-19T21:54:51,892 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T21:54:51,892 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,892 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T21:54:51,892 DEBUG [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T21:54:51,892 DEBUG [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(1351): Waiting on 42590388ea221dcf7856712664520cbd 2024-11-19T21:54:51,892 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,892 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T21:54:51,892 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. after waiting 0 ms 2024-11-19T21:54:51,892 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T21:54:51,892 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T21:54:51,892 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,892 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T21:54:51,892 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-19T21:54:51,892 DEBUG [StoreFileCloser-42590388ea221dcf7856712664520cbd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/family/b469939c07f64a30af910994e0f11f03 2024-11-19T21:54:51,896 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnRegionMove1732053289413/42590388ea221dcf7856712664520cbd/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-19T21:54:51,897 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,897 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 42590388ea221dcf7856712664520cbd: Waiting for close lock at 1732053291891Running coprocessor pre-close hooks at 1732053291892 (+1 ms)Disabling compacts and flushes for region at 1732053291892Disabling writes for close at 1732053291892Writing region close event to WAL at 1732053291893 (+1 ms)Running coprocessor post-close hooks at 1732053291897 (+4 ms)Closed at 1732053291897 2024-11-19T21:54:51,897 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd. 2024-11-19T21:54:51,901 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1f420b92ba644726900bc577df176c71 2024-11-19T21:54:51,914 INFO [regionserver/707262ffa86d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:54:51,915 INFO [regionserver/707262ffa86d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:54:51,916 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/info/1f420b92ba644726900bc577df176c71 is 205, key is testBlockEvictionOnRegionMove1732053289413,,1732053289427.42590388ea221dcf7856712664520cbd./info:regioninfo/1732053291789/Put/seqid=0 2024-11-19T21:54:51,916 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/info/1f420b92ba644726900bc577df176c71 2024-11-19T21:54:51,925 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1035366c8b71447a9186c3b30e047d97 2024-11-19T21:54:51,937 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/ns/1035366c8b71447a9186c3b30e047d97 is 43, key is default/ns:d/1732053289144/Put/seqid=0 2024-11-19T21:54:51,938 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/ns/1035366c8b71447a9186c3b30e047d97 2024-11-19T21:54:51,946 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 21e5eb4f42e440c49285e4962408a869 2024-11-19T21:54:51,960 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/table/21e5eb4f42e440c49285e4962408a869 is 78, key is testBlockEvictionOnRegionMove1732053289413/table:state/1732053289837/Put/seqid=0 2024-11-19T21:54:51,960 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/table/21e5eb4f42e440c49285e4962408a869 2024-11-19T21:54:51,963 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/info/1f420b92ba644726900bc577df176c71 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/info/1f420b92ba644726900bc577df176c71 2024-11-19T21:54:51,964 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/info/1f420b92ba644726900bc577df176c71, delay=980 ms 2024-11-19T21:54:51,965 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/info/1f420b92ba644726900bc577df176c71, entries=15, sequenceid=15, filesize=7.6 K 2024-11-19T21:54:51,966 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/ns/1035366c8b71447a9186c3b30e047d97 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/ns/1035366c8b71447a9186c3b30e047d97 2024-11-19T21:54:51,967 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/ns/1035366c8b71447a9186c3b30e047d97, delay=930 ms 2024-11-19T21:54:51,969 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/ns/1035366c8b71447a9186c3b30e047d97, entries=2, sequenceid=15, filesize=5.0 K 2024-11-19T21:54:51,969 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/table/21e5eb4f42e440c49285e4962408a869 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/table/21e5eb4f42e440c49285e4962408a869 2024-11-19T21:54:51,970 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/table/21e5eb4f42e440c49285e4962408a869, delay=903 ms 2024-11-19T21:54:51,972 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/table/21e5eb4f42e440c49285e4962408a869, entries=2, sequenceid=15, filesize=5.3 K 2024-11-19T21:54:51,972 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=15, compaction requested=false 2024-11-19T21:54:51,973 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T21:54:51,974 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/info/1f420b92ba644726900bc577df176c71 2024-11-19T21:54:51,976 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/ns/1035366c8b71447a9186c3b30e047d97 2024-11-19T21:54:51,977 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/table/21e5eb4f42e440c49285e4962408a869 2024-11-19T21:54:51,981 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-19T21:54:51,981 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T21:54:51,982 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T21:54:51,982 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732053291892Running coprocessor pre-close hooks at 1732053291892Disabling compacts and flushes for region at 1732053291892Disabling writes for close at 1732053291892Obtaining lock to block concurrent updates at 1732053291892Preparing flush snapshotting stores in 1588230740 at 1732053291892Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732053291893 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732053291893Flushing 1588230740/info: creating writer at 1732053291893Flushing 1588230740/info: appending metadata at 1732053291915 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732053291915Flushing 1588230740/ns: creating writer at 1732053291919 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732053291936 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732053291936Flushing 1588230740/table: creating writer at 1732053291940 (+4 ms)Flushing 1588230740/table: appending metadata at 1732053291958 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732053291958Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@396290ea: reopening flushed file at 1732053291963 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fa5c90f: reopening flushed file at 1732053291966 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10de1367: reopening flushed file at 1732053291969 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=15, compaction requested=false at 1732053291973 (+4 ms)Writing region close event to WAL at 1732053291977 (+4 ms)Running coprocessor post-close hooks at 1732053291981 (+4 ms)Closed at 1732053291982 (+1 ms) 2024-11-19T21:54:51,982 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T21:54:52,092 INFO [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(976): stopping server 707262ffa86d,46439,1732053287071; all regions closed. 2024-11-19T21:54:52,092 INFO [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(976): stopping server 707262ffa86d,34667,1732053286895; all regions closed. 2024-11-19T21:54:52,097 DEBUG [RS:1;707262ffa86d:46439 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs 2024-11-19T21:54:52,097 DEBUG [RS:0;707262ffa86d:34667 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs 2024-11-19T21:54:52,098 INFO [RS:1;707262ffa86d:46439 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 707262ffa86d%2C46439%2C1732053287071:(num 1732053288566) 2024-11-19T21:54:52,098 INFO [RS:0;707262ffa86d:34667 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 707262ffa86d%2C34667%2C1732053286895.meta:.meta(num 1732053289012) 2024-11-19T21:54:52,098 DEBUG [RS:1;707262ffa86d:46439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:52,098 INFO [RS:1;707262ffa86d:46439 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:54:52,098 INFO [RS:1;707262ffa86d:46439 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T21:54:52,098 INFO [RS:1;707262ffa86d:46439 {}] hbase.ChoreService(370): Chore service for: regionserver/707262ffa86d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T21:54:52,098 INFO [RS:1;707262ffa86d:46439 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T21:54:52,098 INFO [RS:1;707262ffa86d:46439 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T21:54:52,099 INFO [RS:1;707262ffa86d:46439 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T21:54:52,099 INFO [regionserver/707262ffa86d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T21:54:52,099 INFO [RS:1;707262ffa86d:46439 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T21:54:52,099 INFO [RS:1;707262ffa86d:46439 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46439 2024-11-19T21:54:52,100 DEBUG [RS:0;707262ffa86d:34667 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs 2024-11-19T21:54:52,100 INFO [RS:0;707262ffa86d:34667 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 707262ffa86d%2C34667%2C1732053286895:(num 1732053288566) 2024-11-19T21:54:52,100 DEBUG [RS:0;707262ffa86d:34667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:52,100 INFO [RS:0;707262ffa86d:34667 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:54:52,100 INFO [RS:0;707262ffa86d:34667 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T21:54:52,101 INFO [RS:0;707262ffa86d:34667 {}] hbase.ChoreService(370): Chore service for: regionserver/707262ffa86d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T21:54:52,101 INFO [regionserver/707262ffa86d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T21:54:52,101 INFO [RS:0;707262ffa86d:34667 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T21:54:52,101 INFO [RS:0;707262ffa86d:34667 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34667 2024-11-19T21:54:52,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T21:54:52,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/707262ffa86d,46439,1732053287071 2024-11-19T21:54:52,130 INFO [RS:1;707262ffa86d:46439 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T21:54:52,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/707262ffa86d,34667,1732053286895 2024-11-19T21:54:52,139 INFO [RS:0;707262ffa86d:34667 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T21:54:52,139 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [707262ffa86d,46439,1732053287071] 2024-11-19T21:54:52,155 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/707262ffa86d,46439,1732053287071 already deleted, retry=false 2024-11-19T21:54:52,156 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 707262ffa86d,46439,1732053287071 expired; onlineServers=1 2024-11-19T21:54:52,156 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [707262ffa86d,34667,1732053286895] 2024-11-19T21:54:52,163 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/707262ffa86d,34667,1732053286895 already deleted, retry=false 2024-11-19T21:54:52,164 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 707262ffa86d,34667,1732053286895 expired; onlineServers=0 2024-11-19T21:54:52,164 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '707262ffa86d,33907,1732053286072' ***** 2024-11-19T21:54:52,164 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T21:54:52,164 INFO [M:0;707262ffa86d:33907 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T21:54:52,164 INFO [M:0;707262ffa86d:33907 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T21:54:52,165 DEBUG [M:0;707262ffa86d:33907 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T21:54:52,165 DEBUG [M:0;707262ffa86d:33907 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T21:54:52,165 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T21:54:52,165 DEBUG [master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.large.0-1732053288271 {}] cleaner.HFileCleaner(306): Exit Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.large.0-1732053288271,5,FailOnTimeoutGroup] 2024-11-19T21:54:52,165 DEBUG [master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.small.0-1732053288273 {}] cleaner.HFileCleaner(306): Exit Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.small.0-1732053288273,5,FailOnTimeoutGroup] 2024-11-19T21:54:52,166 INFO [M:0;707262ffa86d:33907 {}] hbase.ChoreService(370): Chore service for: master/707262ffa86d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T21:54:52,166 INFO [M:0;707262ffa86d:33907 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T21:54:52,166 DEBUG [M:0;707262ffa86d:33907 {}] master.HMaster(1795): Stopping service threads 2024-11-19T21:54:52,166 INFO [M:0;707262ffa86d:33907 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T21:54:52,167 INFO [M:0;707262ffa86d:33907 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T21:54:52,168 INFO [M:0;707262ffa86d:33907 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T21:54:52,168 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T21:54:52,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T21:54:52,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,172 DEBUG [M:0;707262ffa86d:33907 {}] zookeeper.ZKUtil(347): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T21:54:52,173 WARN [M:0;707262ffa86d:33907 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T21:54:52,173 INFO [M:0;707262ffa86d:33907 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/.lastflushedseqids 2024-11-19T21:54:52,186 INFO [M:0;707262ffa86d:33907 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T21:54:52,186 INFO [M:0;707262ffa86d:33907 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T21:54:52,187 DEBUG [M:0;707262ffa86d:33907 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T21:54:52,187 INFO [M:0;707262ffa86d:33907 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:52,187 DEBUG [M:0;707262ffa86d:33907 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:52,187 DEBUG [M:0;707262ffa86d:33907 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T21:54:52,187 DEBUG [M:0;707262ffa86d:33907 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:52,187 INFO [M:0;707262ffa86d:33907 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-19T21:54:52,195 DEBUG [M:0;707262ffa86d:33907 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9abdf3830330479c9e9f1272484443a0 2024-11-19T21:54:52,209 DEBUG [M:0;707262ffa86d:33907 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9abdf3830330479c9e9f1272484443a0 is 82, key is hbase:meta,,1/info:regioninfo/1732053289070/Put/seqid=0 2024-11-19T21:54:52,210 INFO [M:0;707262ffa86d:33907 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9abdf3830330479c9e9f1272484443a0 2024-11-19T21:54:52,217 DEBUG [M:0;707262ffa86d:33907 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8480f2ac5fc342feade96dfd1641d0a7 2024-11-19T21:54:52,229 DEBUG [M:0;707262ffa86d:33907 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8480f2ac5fc342feade96dfd1641d0a7 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732053289842/Put/seqid=0 2024-11-19T21:54:52,230 INFO [M:0;707262ffa86d:33907 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8480f2ac5fc342feade96dfd1641d0a7 2024-11-19T21:54:52,236 DEBUG [M:0;707262ffa86d:33907 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 84914bc781bf4d0aa302ab4ccd30da30 2024-11-19T21:54:52,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:52,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x10156614e420002, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:52,247 INFO [RS:1;707262ffa86d:46439 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T21:54:52,248 INFO [RS:1;707262ffa86d:46439 {}] regionserver.HRegionServer(1031): Exiting; stopping=707262ffa86d,46439,1732053287071; zookeeper connection closed. 2024-11-19T21:54:52,248 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@51aae58a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@51aae58a 2024-11-19T21:54:52,248 DEBUG [M:0;707262ffa86d:33907 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84914bc781bf4d0aa302ab4ccd30da30 is 69, key is 707262ffa86d,34667,1732053286895/rs:state/1732053288283/Put/seqid=0 2024-11-19T21:54:52,248 INFO [M:0;707262ffa86d:33907 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84914bc781bf4d0aa302ab4ccd30da30 2024-11-19T21:54:52,251 DEBUG [M:0;707262ffa86d:33907 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9abdf3830330479c9e9f1272484443a0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9abdf3830330479c9e9f1272484443a0 2024-11-19T21:54:52,252 INFO [M:0;707262ffa86d:33907 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9abdf3830330479c9e9f1272484443a0, entries=8, sequenceid=84, filesize=5.5 K 2024-11-19T21:54:52,253 DEBUG [M:0;707262ffa86d:33907 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8480f2ac5fc342feade96dfd1641d0a7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8480f2ac5fc342feade96dfd1641d0a7 2024-11-19T21:54:52,255 INFO [M:0;707262ffa86d:33907 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8480f2ac5fc342feade96dfd1641d0a7, entries=9, sequenceid=84, filesize=6.5 K 2024-11-19T21:54:52,255 DEBUG [M:0;707262ffa86d:33907 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84914bc781bf4d0aa302ab4ccd30da30 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/84914bc781bf4d0aa302ab4ccd30da30 2024-11-19T21:54:52,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:52,255 INFO [RS:0;707262ffa86d:34667 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T21:54:52,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34667-0x10156614e420001, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:52,255 INFO [RS:0;707262ffa86d:34667 {}] regionserver.HRegionServer(1031): Exiting; stopping=707262ffa86d,34667,1732053286895; zookeeper connection closed. 2024-11-19T21:54:52,256 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1fb76b1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1fb76b1 2024-11-19T21:54:52,256 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-19T21:54:52,257 INFO [M:0;707262ffa86d:33907 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/84914bc781bf4d0aa302ab4ccd30da30, entries=2, sequenceid=84, filesize=5.1 K 2024-11-19T21:54:52,258 INFO [M:0;707262ffa86d:33907 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=84, compaction requested=false 2024-11-19T21:54:52,260 INFO [M:0;707262ffa86d:33907 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:52,260 DEBUG [M:0;707262ffa86d:33907 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732053292186Disabling compacts and flushes for region at 1732053292186Disabling writes for close at 1732053292187 (+1 ms)Obtaining lock to block concurrent updates at 1732053292187Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732053292187Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732053292188 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732053292188Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732053292188Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732053292208 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732053292208Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732053292211 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732053292228 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732053292229 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732053292231 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732053292247 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732053292248 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75123694: reopening flushed file at 1732053292250 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@211638df: reopening flushed file at 1732053292253 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6526368b: reopening flushed file at 1732053292255 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=84, compaction requested=false at 1732053292258 (+3 ms)Writing region close event to WAL at 1732053292260 (+2 ms)Closed at 1732053292260 2024-11-19T21:54:52,261 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T21:54:52,261 INFO [M:0;707262ffa86d:33907 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T21:54:52,261 INFO [M:0;707262ffa86d:33907 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33907 2024-11-19T21:54:52,262 INFO [M:0;707262ffa86d:33907 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T21:54:52,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:52,388 INFO [M:0;707262ffa86d:33907 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T21:54:52,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33907-0x10156614e420000, quorum=127.0.0.1:55554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:52,396 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T21:54:52,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-19T21:54:52,416 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:52,423 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/707262ffa86d:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/707262ffa86d:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732053291967 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732053291774 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/707262ffa86d:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732053291964 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732053290224 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/707262ffa86d:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=402 (was 394) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5506 (was 5888) 2024-11-19T21:54:52,428 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=402, ProcessCount=11, AvailableMemoryMB=5506 2024-11-19T21:54:52,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/hadoop-log-dir so I do NOT create it in target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d 2024-11-19T21:54:52,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/hadoop-tmp-dir so I do NOT create it in target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d 2024-11-19T21:54:52,429 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:52,431 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/cluster_87d210ce-e040-bd4a-cd91-dabe9b6e74da, deleteOnExit=true 2024-11-19T21:54:52,434 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/cluster_87d210ce-e040-bd4a-cd91-dabe9b6e74da/zookeeper_0, clientPort=57934, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/cluster_87d210ce-e040-bd4a-cd91-dabe9b6e74da/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/cluster_87d210ce-e040-bd4a-cd91-dabe9b6e74da/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T21:54:52,435 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57934 2024-11-19T21:54:52,448 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab with version=8 2024-11-19T21:54:52,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/hbase-staging 2024-11-19T21:54:52,451 INFO [Time-limited test {}] client.ConnectionUtils(128): master/707262ffa86d:0 server-side Connection retries=45 2024-11-19T21:54:52,451 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:52,452 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:52,452 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T21:54:52,452 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:52,452 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T21:54:52,452 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T21:54:52,452 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T21:54:52,453 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41449 2024-11-19T21:54:52,455 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41449 connecting to ZooKeeper ensemble=127.0.0.1:57934 2024-11-19T21:54:52,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:414490x0, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T21:54:52,488 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41449-0x101566168560000 connected 2024-11-19T21:54:52,572 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:52,573 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:52,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:52,575 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab, hbase.cluster.distributed=false 2024-11-19T21:54:52,579 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T21:54:52,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41449 2024-11-19T21:54:52,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41449 2024-11-19T21:54:52,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41449 2024-11-19T21:54:52,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41449 2024-11-19T21:54:52,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41449 2024-11-19T21:54:52,597 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/707262ffa86d:0 server-side Connection retries=45 2024-11-19T21:54:52,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:52,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:52,597 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T21:54:52,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:52,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T21:54:52,597 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T21:54:52,598 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T21:54:52,598 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36609 2024-11-19T21:54:52,600 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36609 connecting to ZooKeeper ensemble=127.0.0.1:57934 2024-11-19T21:54:52,600 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:52,600 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:52,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366090x0, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T21:54:52,613 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36609-0x101566168560001 connected 2024-11-19T21:54:52,613 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:52,614 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T21:54:52,614 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.cache 2024-11-19T21:54:52,615 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T21:54:52,615 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T21:54:52,615 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T21:54:52,615 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T21:54:52,615 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T21:54:52,615 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T21:54:52,616 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T21:54:52,616 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T21:54:52,616 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.persistence is complete. 2024-11-19T21:54:52,617 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T21:54:52,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T21:54:52,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36609 2024-11-19T21:54:52,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36609 2024-11-19T21:54:52,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36609 2024-11-19T21:54:52,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36609 2024-11-19T21:54:52,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36609 2024-11-19T21:54:52,644 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/707262ffa86d:0 server-side Connection retries=45 2024-11-19T21:54:52,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:52,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:52,644 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T21:54:52,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:52,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T21:54:52,644 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T21:54:52,645 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T21:54:52,645 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46853 2024-11-19T21:54:52,647 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46853 connecting to ZooKeeper ensemble=127.0.0.1:57934 2024-11-19T21:54:52,647 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:52,648 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:52,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:468530x0, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T21:54:52,656 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:52,656 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46853-0x101566168560002 connected 2024-11-19T21:54:52,656 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T21:54:52,657 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.cache 2024-11-19T21:54:52,657 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T21:54:52,658 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T21:54:52,658 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T21:54:52,658 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T21:54:52,658 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T21:54:52,658 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T21:54:52,658 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T21:54:52,658 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T21:54:52,659 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.persistence is complete. 2024-11-19T21:54:52,659 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T21:54:52,660 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T21:54:52,661 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46853 2024-11-19T21:54:52,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46853 2024-11-19T21:54:52,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46853 2024-11-19T21:54:52,667 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46853 2024-11-19T21:54:52,667 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46853 2024-11-19T21:54:52,678 DEBUG [M:0;707262ffa86d:41449 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;707262ffa86d:41449 2024-11-19T21:54:52,679 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/707262ffa86d,41449,1732053292450 2024-11-19T21:54:52,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:52,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:52,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:52,689 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/707262ffa86d,41449,1732053292450 2024-11-19T21:54:52,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T21:54:52,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T21:54:52,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,698 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T21:54:52,698 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/707262ffa86d,41449,1732053292450 from backup master directory 2024-11-19T21:54:52,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:52,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/707262ffa86d,41449,1732053292450 2024-11-19T21:54:52,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:52,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:54:52,705 WARN [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T21:54:52,705 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=707262ffa86d,41449,1732053292450 2024-11-19T21:54:52,706 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/hbase.id] with ID: 3ecd411f-96b3-4243-9240-1634950a407d 2024-11-19T21:54:52,706 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/.tmp/hbase.id 2024-11-19T21:54:52,709 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/hbase.id] 2024-11-19T21:54:52,744 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:52,744 INFO [master/707262ffa86d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T21:54:52,747 INFO [master/707262ffa86d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-19T21:54:52,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,760 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:54:52,761 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T21:54:52,762 INFO [master/707262ffa86d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:54:52,782 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store 2024-11-19T21:54:52,791 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:52,791 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T21:54:52,791 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:52,791 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:52,791 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T21:54:52,791 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:52,791 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:52,791 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732053292791Disabling compacts and flushes for region at 1732053292791Disabling writes for close at 1732053292791Writing region close event to WAL at 1732053292791Closed at 1732053292791 2024-11-19T21:54:52,792 WARN [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/.initializing 2024-11-19T21:54:52,792 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/WALs/707262ffa86d,41449,1732053292450 2024-11-19T21:54:52,797 INFO [master/707262ffa86d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T21:54:52,801 INFO [master/707262ffa86d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=707262ffa86d%2C41449%2C1732053292450, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/WALs/707262ffa86d,41449,1732053292450, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/oldWALs, maxLogs=10 2024-11-19T21:54:52,819 INFO [master/707262ffa86d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/WALs/707262ffa86d,41449,1732053292450/707262ffa86d%2C41449%2C1732053292450.1732053292801 2024-11-19T21:54:52,819 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T21:54:52,819 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:54:52,819 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:52,819 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,819 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,824 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T21:54:52,824 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:52,824 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:52,824 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T21:54:52,828 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:52,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:54:52,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T21:54:52,831 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:52,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:54:52,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T21:54:52,835 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:52,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:54:52,835 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,835 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,836 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,836 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,836 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,836 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T21:54:52,836 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:54:52,841 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:54:52,842 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64810259, jitterRate=-0.03425188362598419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T21:54:52,842 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732053292819Initializing all the Stores at 1732053292820 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053292820Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053292820Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053292820Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053292820Cleaning up temporary data from old regions at 1732053292836 (+16 ms)Region opened successfully at 1732053292842 (+6 ms) 2024-11-19T21:54:52,843 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T21:54:52,846 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b57de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=707262ffa86d/172.17.0.2:0 2024-11-19T21:54:52,848 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T21:54:52,848 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T21:54:52,848 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T21:54:52,848 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T21:54:52,848 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T21:54:52,849 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-19T21:54:52,849 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T21:54:52,851 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T21:54:52,852 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T21:54:52,863 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T21:54:52,864 INFO [master/707262ffa86d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T21:54:52,864 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T21:54:52,871 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T21:54:52,872 INFO [master/707262ffa86d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T21:54:52,873 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T21:54:52,880 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T21:54:52,881 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T21:54:52,888 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T21:54:52,892 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T21:54:52,897 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T21:54:52,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:52,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:52,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:52,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,906 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=707262ffa86d,41449,1732053292450, sessionid=0x101566168560000, setting cluster-up flag (Was=false) 2024-11-19T21:54:52,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,947 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T21:54:52,949 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=707262ffa86d,41449,1732053292450 2024-11-19T21:54:52,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:52,988 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T21:54:52,990 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=707262ffa86d,41449,1732053292450 2024-11-19T21:54:52,990 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T21:54:52,992 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T21:54:52,992 INFO [master/707262ffa86d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T21:54:52,992 INFO [master/707262ffa86d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T21:54:52,992 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 707262ffa86d,41449,1732053292450 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T21:54:52,994 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:54:52,994 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:54:52,994 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:54:52,994 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:54:52,994 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/707262ffa86d:0, corePoolSize=10, maxPoolSize=10 2024-11-19T21:54:52,995 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:52,995 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/707262ffa86d:0, corePoolSize=2, maxPoolSize=2 2024-11-19T21:54:52,995 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:52,998 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732053322998 2024-11-19T21:54:52,998 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T21:54:52,998 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T21:54:52,998 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T21:54:52,998 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T21:54:52,998 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T21:54:52,998 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T21:54:52,998 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T21:54:52,998 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T21:54:52,999 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:52,999 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T21:54:52,999 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,000 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T21:54:53,000 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T21:54:53,000 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T21:54:53,003 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T21:54:53,003 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T21:54:53,003 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.large.0-1732053293003,5,FailOnTimeoutGroup] 2024-11-19T21:54:53,003 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.small.0-1732053293003,5,FailOnTimeoutGroup] 2024-11-19T21:54:53,003 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,003 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T21:54:53,003 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,003 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,021 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T21:54:53,022 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab 2024-11-19T21:54:53,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:53,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T21:54:53,034 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T21:54:53,034 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:53,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:53,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T21:54:53,038 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T21:54:53,038 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:53,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:53,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T21:54:53,042 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T21:54:53,042 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:53,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:53,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T21:54:53,045 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T21:54:53,045 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:53,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:53,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T21:54:53,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740 2024-11-19T21:54:53,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740 2024-11-19T21:54:53,047 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T21:54:53,047 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T21:54:53,047 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T21:54:53,047 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T21:54:53,053 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:54:53,054 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73495408, jitterRate=0.09516692161560059}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T21:54:53,054 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732053293030Initializing all the Stores at 1732053293030Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053293030Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053293030Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053293031 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053293031Cleaning up temporary data from old regions at 1732053293047 (+16 ms)Region opened successfully at 1732053293054 (+7 ms) 2024-11-19T21:54:53,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T21:54:53,055 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T21:54:53,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T21:54:53,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T21:54:53,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T21:54:53,056 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T21:54:53,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732053293055Disabling compacts and flushes for region at 1732053293055Disabling writes for close at 1732053293055Writing region close event to WAL at 1732053293056 (+1 ms)Closed at 1732053293056 2024-11-19T21:54:53,057 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T21:54:53,057 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T21:54:53,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T21:54:53,058 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T21:54:53,059 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T21:54:53,070 INFO [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(746): ClusterId : 3ecd411f-96b3-4243-9240-1634950a407d 2024-11-19T21:54:53,070 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(746): ClusterId : 3ecd411f-96b3-4243-9240-1634950a407d 2024-11-19T21:54:53,070 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T21:54:53,070 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T21:54:53,097 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T21:54:53,097 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T21:54:53,097 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T21:54:53,097 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T21:54:53,106 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T21:54:53,106 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T21:54:53,107 DEBUG [RS:0;707262ffa86d:36609 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f10ad0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=707262ffa86d/172.17.0.2:0 2024-11-19T21:54:53,107 DEBUG [RS:1;707262ffa86d:46853 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c107614, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=707262ffa86d/172.17.0.2:0 2024-11-19T21:54:53,118 DEBUG [RS:1;707262ffa86d:46853 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;707262ffa86d:46853 2024-11-19T21:54:53,118 DEBUG [RS:0;707262ffa86d:36609 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;707262ffa86d:36609 2024-11-19T21:54:53,118 INFO [RS:1;707262ffa86d:46853 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T21:54:53,119 INFO [RS:1;707262ffa86d:46853 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T21:54:53,119 INFO [RS:0;707262ffa86d:36609 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T21:54:53,119 DEBUG [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T21:54:53,119 INFO [RS:0;707262ffa86d:36609 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T21:54:53,119 DEBUG [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T21:54:53,120 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(2659): reportForDuty to master=707262ffa86d,41449,1732053292450 with port=36609, startcode=1732053292597 2024-11-19T21:54:53,120 INFO [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(2659): reportForDuty to master=707262ffa86d,41449,1732053292450 with port=46853, startcode=1732053292643 2024-11-19T21:54:53,120 DEBUG [RS:0;707262ffa86d:36609 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T21:54:53,120 DEBUG [RS:1;707262ffa86d:46853 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T21:54:53,122 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43543, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T21:54:53,122 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52885, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T21:54:53,123 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41449 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 707262ffa86d,46853,1732053292643 2024-11-19T21:54:53,123 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41449 {}] master.ServerManager(517): Registering regionserver=707262ffa86d,46853,1732053292643 2024-11-19T21:54:53,124 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41449 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 707262ffa86d,36609,1732053292597 2024-11-19T21:54:53,125 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41449 {}] master.ServerManager(517): Registering regionserver=707262ffa86d,36609,1732053292597 2024-11-19T21:54:53,125 DEBUG [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab 2024-11-19T21:54:53,125 DEBUG [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T21:54:53,125 DEBUG [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T21:54:53,126 DEBUG [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab 2024-11-19T21:54:53,126 DEBUG [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T21:54:53,126 DEBUG [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T21:54:53,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T21:54:53,197 DEBUG [RS:1;707262ffa86d:46853 {}] zookeeper.ZKUtil(111): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/707262ffa86d,46853,1732053292643 2024-11-19T21:54:53,197 WARN [RS:1;707262ffa86d:46853 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T21:54:53,197 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [707262ffa86d,46853,1732053292643] 2024-11-19T21:54:53,197 INFO [RS:1;707262ffa86d:46853 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:54:53,197 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [707262ffa86d,36609,1732053292597] 2024-11-19T21:54:53,197 DEBUG [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,46853,1732053292643 2024-11-19T21:54:53,197 DEBUG [RS:0;707262ffa86d:36609 {}] zookeeper.ZKUtil(111): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/707262ffa86d,36609,1732053292597 2024-11-19T21:54:53,198 WARN [RS:0;707262ffa86d:36609 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T21:54:53,198 INFO [RS:0;707262ffa86d:36609 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:54:53,198 DEBUG [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,36609,1732053292597 2024-11-19T21:54:53,205 INFO [RS:1;707262ffa86d:46853 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T21:54:53,206 INFO [RS:0;707262ffa86d:36609 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T21:54:53,208 INFO [RS:1;707262ffa86d:46853 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T21:54:53,209 INFO [RS:1;707262ffa86d:46853 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T21:54:53,209 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,209 WARN [707262ffa86d:41449 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-19T21:54:53,209 INFO [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T21:54:53,210 INFO [RS:1;707262ffa86d:46853 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T21:54:53,210 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,210 INFO [RS:0;707262ffa86d:36609 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T21:54:53,211 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,211 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,211 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,211 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,211 INFO [RS:0;707262ffa86d:36609 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T21:54:53,211 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,211 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,211 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/707262ffa86d:0, corePoolSize=2, maxPoolSize=2 2024-11-19T21:54:53,212 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,212 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T21:54:53,212 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,212 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,212 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,212 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,212 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,212 INFO [RS:0;707262ffa86d:36609 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T21:54:53,212 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:54:53,212 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,212 DEBUG [RS:1;707262ffa86d:46853 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:54:53,212 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,212 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,213 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,213 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,213 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,213 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,213 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,213 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,213 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,213 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/707262ffa86d:0, corePoolSize=2, maxPoolSize=2 2024-11-19T21:54:53,213 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,213 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,213 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,46853,1732053292643-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T21:54:53,213 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,213 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,213 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,213 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,213 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:53,214 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:54:53,214 DEBUG [RS:0;707262ffa86d:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:54:53,214 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,214 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,215 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,215 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,215 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,215 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,36609,1732053292597-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T21:54:53,229 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T21:54:53,229 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,36609,1732053292597-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,229 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,229 INFO [RS:0;707262ffa86d:36609 {}] regionserver.Replication(171): 707262ffa86d,36609,1732053292597 started 2024-11-19T21:54:53,230 INFO [RS:1;707262ffa86d:46853 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T21:54:53,230 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,46853,1732053292643-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,230 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,230 INFO [RS:1;707262ffa86d:46853 {}] regionserver.Replication(171): 707262ffa86d,46853,1732053292643 started 2024-11-19T21:54:53,243 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,243 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(1482): Serving as 707262ffa86d,36609,1732053292597, RpcServer on 707262ffa86d/172.17.0.2:36609, sessionid=0x101566168560001 2024-11-19T21:54:53,243 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T21:54:53,243 DEBUG [RS:0;707262ffa86d:36609 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 707262ffa86d,36609,1732053292597 2024-11-19T21:54:53,243 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,36609,1732053292597' 2024-11-19T21:54:53,243 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T21:54:53,244 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T21:54:53,245 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T21:54:53,245 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T21:54:53,245 DEBUG [RS:0;707262ffa86d:36609 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 707262ffa86d,36609,1732053292597 2024-11-19T21:54:53,245 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,36609,1732053292597' 2024-11-19T21:54:53,245 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T21:54:53,245 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T21:54:53,246 DEBUG [RS:0;707262ffa86d:36609 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T21:54:53,246 INFO [RS:0;707262ffa86d:36609 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T21:54:53,246 INFO [RS:0;707262ffa86d:36609 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T21:54:53,248 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,248 INFO [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(1482): Serving as 707262ffa86d,46853,1732053292643, RpcServer on 707262ffa86d/172.17.0.2:46853, sessionid=0x101566168560002 2024-11-19T21:54:53,248 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T21:54:53,248 DEBUG [RS:1;707262ffa86d:46853 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 707262ffa86d,46853,1732053292643 2024-11-19T21:54:53,248 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,46853,1732053292643' 2024-11-19T21:54:53,248 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T21:54:53,249 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T21:54:53,250 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T21:54:53,250 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T21:54:53,250 DEBUG [RS:1;707262ffa86d:46853 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 707262ffa86d,46853,1732053292643 2024-11-19T21:54:53,250 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,46853,1732053292643' 2024-11-19T21:54:53,250 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T21:54:53,250 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T21:54:53,251 DEBUG [RS:1;707262ffa86d:46853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T21:54:53,251 INFO [RS:1;707262ffa86d:46853 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T21:54:53,251 INFO [RS:1;707262ffa86d:46853 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T21:54:53,347 INFO [RS:0;707262ffa86d:36609 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T21:54:53,348 INFO [RS:0;707262ffa86d:36609 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=707262ffa86d%2C36609%2C1732053292597, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,36609,1732053292597, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs, maxLogs=55 2024-11-19T21:54:53,352 INFO [RS:1;707262ffa86d:46853 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T21:54:53,353 INFO [RS:1;707262ffa86d:46853 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=707262ffa86d%2C46853%2C1732053292643, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,46853,1732053292643, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs, maxLogs=55 2024-11-19T21:54:53,371 INFO [RS:0;707262ffa86d:36609 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,36609,1732053292597/707262ffa86d%2C36609%2C1732053292597.1732053293351 2024-11-19T21:54:53,373 DEBUG [RS:0;707262ffa86d:36609 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T21:54:53,376 INFO [RS:1;707262ffa86d:46853 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,46853,1732053292643/707262ffa86d%2C46853%2C1732053292643.1732053293354 2024-11-19T21:54:53,376 DEBUG [RS:1;707262ffa86d:46853 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T21:54:53,460 DEBUG [707262ffa86d:41449 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-19T21:54:53,460 DEBUG [707262ffa86d:41449 {}] balancer.BalancerClusterState(204): Hosts are {707262ffa86d=0} racks are {/default-rack=0} 2024-11-19T21:54:53,462 DEBUG [707262ffa86d:41449 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T21:54:53,462 DEBUG [707262ffa86d:41449 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T21:54:53,462 DEBUG [707262ffa86d:41449 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T21:54:53,462 DEBUG [707262ffa86d:41449 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T21:54:53,462 INFO [707262ffa86d:41449 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T21:54:53,462 INFO [707262ffa86d:41449 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T21:54:53,462 DEBUG [707262ffa86d:41449 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T21:54:53,463 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=707262ffa86d,36609,1732053292597 2024-11-19T21:54:53,465 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 707262ffa86d,36609,1732053292597, state=OPENING 2024-11-19T21:54:53,488 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T21:54:53,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:53,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:53,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:53,498 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T21:54:53,498 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:53,498 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:53,498 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:53,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=707262ffa86d,36609,1732053292597}] 2024-11-19T21:54:53,651 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T21:54:53,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51161, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T21:54:53,656 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T21:54:53,656 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:54:53,656 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T21:54:53,657 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=707262ffa86d%2C36609%2C1732053292597.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,36609,1732053292597, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs, maxLogs=55 2024-11-19T21:54:53,676 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,36609,1732053292597/707262ffa86d%2C36609%2C1732053292597.meta.1732053293658.meta 2024-11-19T21:54:53,677 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T21:54:53,677 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:54:53,677 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T21:54:53,677 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T21:54:53,678 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T21:54:53,678 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T21:54:53,678 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:53,678 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T21:54:53,678 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T21:54:53,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T21:54:53,680 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T21:54:53,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:53,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:53,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T21:54:53,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T21:54:53,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:53,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:53,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T21:54:53,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T21:54:53,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:53,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:53,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T21:54:53,682 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T21:54:53,682 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:53,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:54:53,682 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T21:54:53,683 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740 2024-11-19T21:54:53,684 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740 2024-11-19T21:54:53,684 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T21:54:53,684 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T21:54:53,684 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T21:54:53,685 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T21:54:53,686 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69662859, jitterRate=0.038057491183280945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T21:54:53,686 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T21:54:53,686 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732053293678Writing region info on filesystem at 1732053293678Initializing all the Stores at 1732053293679 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053293679Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053293679Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053293679Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732053293679Cleaning up temporary data from old regions at 1732053293684 (+5 ms)Running coprocessor post-open hooks at 1732053293686 (+2 ms)Region opened successfully at 1732053293686 2024-11-19T21:54:53,687 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732053293651 2024-11-19T21:54:53,689 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T21:54:53,690 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T21:54:53,690 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=707262ffa86d,36609,1732053292597 2024-11-19T21:54:53,691 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 707262ffa86d,36609,1732053292597, state=OPEN 2024-11-19T21:54:53,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T21:54:53,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T21:54:53,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T21:54:53,713 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=707262ffa86d,36609,1732053292597 2024-11-19T21:54:53,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:53,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:53,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:54:53,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T21:54:53,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=707262ffa86d,36609,1732053292597 in 215 msec 2024-11-19T21:54:53,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T21:54:53,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 659 msec 2024-11-19T21:54:53,720 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T21:54:53,720 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T21:54:53,724 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:54:53,724 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,36609,1732053292597, seqNum=-1] 2024-11-19T21:54:53,725 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:54:53,727 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48451, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:54:53,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 741 msec 2024-11-19T21:54:53,734 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732053293734, completionTime=-1 2024-11-19T21:54:53,734 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-19T21:54:53,734 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-19T21:54:53,737 INFO [master/707262ffa86d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-19T21:54:53,737 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732053353737 2024-11-19T21:54:53,737 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732053413737 2024-11-19T21:54:53,738 INFO [master/707262ffa86d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-19T21:54:53,738 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,41449,1732053292450-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,738 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,41449,1732053292450-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,738 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,41449,1732053292450-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,738 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-707262ffa86d:41449, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,738 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,739 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,741 DEBUG [master/707262ffa86d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T21:54:53,744 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.039sec 2024-11-19T21:54:53,745 INFO [master/707262ffa86d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T21:54:53,745 INFO [master/707262ffa86d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T21:54:53,745 INFO [master/707262ffa86d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T21:54:53,745 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T21:54:53,745 INFO [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T21:54:53,745 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,41449,1732053292450-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T21:54:53,745 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,41449,1732053292450-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T21:54:53,746 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T21:54:53,746 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T21:54:53,746 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,41449,1732053292450-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:53,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2129cce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:54:53,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,41449,-1 for getting cluster id 2024-11-19T21:54:53,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:54:53,772 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3ecd411f-96b3-4243-9240-1634950a407d' 2024-11-19T21:54:53,773 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:54:53,773 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3ecd411f-96b3-4243-9240-1634950a407d" 2024-11-19T21:54:53,773 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3e9287, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:54:53,773 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,41449,-1] 2024-11-19T21:54:53,774 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:54:53,774 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:53,776 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:54:53,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:54:53,778 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:54:53,779 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,36609,1732053292597, seqNum=-1] 2024-11-19T21:54:53,779 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:54:53,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:54:53,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=707262ffa86d,41449,1732053292450 2024-11-19T21:54:53,784 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T21:54:53,786 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 707262ffa86d,41449,1732053292450 2024-11-19T21:54:53,786 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3aba8e80 2024-11-19T21:54:53,786 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T21:54:53,788 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T21:54:53,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41449 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732053293784', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:54:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41449 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732053293784 2024-11-19T21:54:53,792 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732053293784 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T21:54:53,793 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:53,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41449 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732053293784" procId is: 4 2024-11-19T21:54:53,794 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732053293784 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T21:54:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:54:53,804 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732053293784-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c9c0b8ee4796868e2123ff071b738206, NAME => 'testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732053293784', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab 2024-11-19T21:54:53,810 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732053293784-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:53,810 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732053293784-pool-0 {}] regionserver.HRegion(1722): Closing c9c0b8ee4796868e2123ff071b738206, disabling compactions & flushes 2024-11-19T21:54:53,810 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732053293784-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:53,810 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732053293784-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:53,810 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732053293784-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. after waiting 0 ms 2024-11-19T21:54:53,810 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732053293784-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:53,810 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732053293784-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:53,810 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732053293784-pool-0 {}] regionserver.HRegion(1676): Region close journal for c9c0b8ee4796868e2123ff071b738206: Waiting for close lock at 1732053293810Disabling compacts and flushes for region at 1732053293810Disabling writes for close at 1732053293810Writing region close event to WAL at 1732053293810Closed at 1732053293810 2024-11-19T21:54:53,812 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732053293784 execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T21:54:53,812 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732053293812"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732053293812"}]},"ts":"1732053293812"} 2024-11-19T21:54:53,815 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T21:54:53,817 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732053293784 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T21:54:53,817 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732053293784","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732053293817"}]},"ts":"1732053293817"} 2024-11-19T21:54:53,819 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732053293784, state=ENABLING in hbase:meta 2024-11-19T21:54:53,820 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {707262ffa86d=0} racks are {/default-rack=0} 2024-11-19T21:54:53,820 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T21:54:53,820 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T21:54:53,820 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T21:54:53,820 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T21:54:53,820 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T21:54:53,820 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T21:54:53,820 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T21:54:53,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732053293784, region=c9c0b8ee4796868e2123ff071b738206, ASSIGN}] 2024-11-19T21:54:53,822 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732053293784, region=c9c0b8ee4796868e2123ff071b738206, ASSIGN 2024-11-19T21:54:53,822 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732053293784, region=c9c0b8ee4796868e2123ff071b738206, ASSIGN; state=OFFLINE, location=707262ffa86d,46853,1732053292643; forceNewPlan=false, retain=false 2024-11-19T21:54:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:54:53,973 INFO [707262ffa86d:41449 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T21:54:53,974 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c9c0b8ee4796868e2123ff071b738206, regionState=OPENING, regionLocation=707262ffa86d,46853,1732053292643 2024-11-19T21:54:53,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732053293784, region=c9c0b8ee4796868e2123ff071b738206, ASSIGN because future has completed 2024-11-19T21:54:53,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9c0b8ee4796868e2123ff071b738206, server=707262ffa86d,46853,1732053292643}] 2024-11-19T21:54:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:54:54,135 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T21:54:54,137 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49399, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T21:54:54,138 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:54,139 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c9c0b8ee4796868e2123ff071b738206, NAME => 'testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:54:54,139 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732053293784 c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:54,139 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:54,139 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:54,139 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:54,140 INFO [StoreOpener-c9c0b8ee4796868e2123ff071b738206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:54,145 INFO [StoreOpener-c9c0b8ee4796868e2123ff071b738206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9c0b8ee4796868e2123ff071b738206 columnFamilyName family 2024-11-19T21:54:54,145 DEBUG [StoreOpener-c9c0b8ee4796868e2123ff071b738206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:54,145 INFO [StoreOpener-c9c0b8ee4796868e2123ff071b738206-1 {}] regionserver.HStore(327): Store=c9c0b8ee4796868e2123ff071b738206/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:54:54,145 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:54,146 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:54,146 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:54,147 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:54,147 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:54,147 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:54,154 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:54:54,154 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c9c0b8ee4796868e2123ff071b738206; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73251309, jitterRate=0.09152956306934357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T21:54:54,154 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:54,155 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c9c0b8ee4796868e2123ff071b738206: Running coprocessor pre-open hook at 1732053294140Writing region info on filesystem at 1732053294140Initializing all the Stores at 1732053294140Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053294140Cleaning up temporary data from old regions at 1732053294147 (+7 ms)Running coprocessor post-open hooks at 1732053294154 (+7 ms)Region opened successfully at 1732053294155 (+1 ms) 2024-11-19T21:54:54,156 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206., pid=6, masterSystemTime=1732053294135 2024-11-19T21:54:54,159 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:54,159 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:54,159 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c9c0b8ee4796868e2123ff071b738206, regionState=OPEN, openSeqNum=2, regionLocation=707262ffa86d,46853,1732053292643 2024-11-19T21:54:54,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9c0b8ee4796868e2123ff071b738206, server=707262ffa86d,46853,1732053292643 because future has completed 2024-11-19T21:54:54,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T21:54:54,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c9c0b8ee4796868e2123ff071b738206, server=707262ffa86d,46853,1732053292643 in 182 msec 2024-11-19T21:54:54,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T21:54:54,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732053293784, region=c9c0b8ee4796868e2123ff071b738206, ASSIGN in 345 msec 2024-11-19T21:54:54,167 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732053293784 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T21:54:54,168 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732053293784","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732053294167"}]},"ts":"1732053294167"} 2024-11-19T21:54:54,170 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732053293784, state=ENABLED in hbase:meta 2024-11-19T21:54:54,171 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732053293784 execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T21:54:54,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732053293784 in 381 msec 2024-11-19T21:54:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:54:54,421 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732053293784 completed 2024-11-19T21:54:54,421 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732053293784 get assigned. Timeout = 60000ms 2024-11-19T21:54:54,421 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T21:54:54,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732053293784 assigned to meta. Checking AM states. 2024-11-19T21:54:54,430 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T21:54:54,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732053293784 assigned. 2024-11-19T21:54:54,434 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732053293784', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206., hostname=707262ffa86d,46853,1732053292643, seqNum=2] 2024-11-19T21:54:54,435 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:54:54,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:54:54,440 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c9c0b8ee4796868e2123ff071b738206 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-19T21:54:54,448 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f8a8bae8822b4cf793c320938f3316b7 2024-11-19T21:54:54,460 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/.tmp/family/f8a8bae8822b4cf793c320938f3316b7 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-19T21:54:54,461 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/.tmp/family/f8a8bae8822b4cf793c320938f3316b7 2024-11-19T21:54:54,463 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/.tmp/family/f8a8bae8822b4cf793c320938f3316b7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/family/f8a8bae8822b4cf793c320938f3316b7 2024-11-19T21:54:54,464 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/family/f8a8bae8822b4cf793c320938f3316b7, delay=932 ms 2024-11-19T21:54:54,465 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/family/f8a8bae8822b4cf793c320938f3316b7, entries=2, sequenceid=6, filesize=4.9 K 2024-11-19T21:54:54,465 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for c9c0b8ee4796868e2123ff071b738206 in 25ms, sequenceid=6, compaction requested=false 2024-11-19T21:54:54,465 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732053293784' 2024-11-19T21:54:54,466 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c9c0b8ee4796868e2123ff071b738206: 2024-11-19T21:54:54,546 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T21:54:55,396 DEBUG [hfile-prefetch-1732053291774 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f8a8bae8822b4cf793c320938f3316b7_0. 2024-11-19T21:54:55,397 DEBUG [hfile-prefetch-1732053291774 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f8a8bae8822b4cf793c320938f3316b7_117. 2024-11-19T21:54:55,397 DEBUG [hfile-prefetch-1732053291774 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/family/f8a8bae8822b4cf793c320938f3316b7, with total blocks 2, and data blocks 1 2024-11-19T21:54:55,397 DEBUG [hfile-prefetch-1732053291774 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-19T21:54:55,397 DEBUG [hfile-prefetch-1732053291774 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/family/f8a8bae8822b4cf793c320938f3316b7, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-19T21:54:55,397 DEBUG [hfile-prefetch-1732053291774 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/family/f8a8bae8822b4cf793c320938f3316b7. Will sleep 100ms and try the verification again. 2024-11-19T21:54:55,466 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;707262ffa86d:46853,5,FailOnTimeoutGroup] 2024-11-19T21:54:55,467 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '707262ffa86d,46853,1732053292643' ***** 2024-11-19T21:54:55,467 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-19T21:54:55,467 INFO [RS:1;707262ffa86d:46853 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T21:54:55,472 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T21:54:55,472 INFO [RS:1;707262ffa86d:46853 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.persistence 2024-11-19T21:54:55,472 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T21:54:55,472 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T21:54:55,472 INFO [RS:1;707262ffa86d:46853 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T21:54:55,472 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T21:54:55,482 DEBUG [RS:1;707262ffa86d:46853 {}] bucket.BucketCache(1487): Thread RS:1;707262ffa86d:46853 started persisting bucket cache to file 2024-11-19T21:54:55,482 DEBUG [RS:1;707262ffa86d:46853 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T21:54:55,482 DEBUG [RS:1;707262ffa86d:46853 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T21:54:55,485 DEBUG [RS:1;707262ffa86d:46853 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-19T21:54:55,485 DEBUG [RS:1;707262ffa86d:46853 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732053295482 2024-11-19T21:54:55,485 DEBUG [RS:1;707262ffa86d:46853 {}] bucket.BucketCache(1509): Thread RS:1;707262ffa86d:46853 finished persisting bucket cache to file, renaming 2024-11-19T21:54:55,485 INFO [RS:1;707262ffa86d:46853 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T21:54:55,485 INFO [RS:1;707262ffa86d:46853 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T21:54:55,485 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T21:54:55,486 INFO [RS:1;707262ffa86d:46853 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T21:54:55,486 INFO [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(3091): Received CLOSE for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:55,486 INFO [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(959): stopping server 707262ffa86d,46853,1732053292643 2024-11-19T21:54:55,486 INFO [RS:1;707262ffa86d:46853 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T21:54:55,486 INFO [RS:1;707262ffa86d:46853 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;707262ffa86d:46853. 2024-11-19T21:54:55,486 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T21:54:55,486 DEBUG [RS:1;707262ffa86d:46853 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:54:55,486 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c9c0b8ee4796868e2123ff071b738206, disabling compactions & flushes 2024-11-19T21:54:55,486 DEBUG [RS:1;707262ffa86d:46853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:55,486 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:55,486 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:55,486 INFO [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T21:54:55,486 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. after waiting 0 ms 2024-11-19T21:54:55,486 DEBUG [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(1325): Online Regions={c9c0b8ee4796868e2123ff071b738206=testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206.} 2024-11-19T21:54:55,486 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:55,486 DEBUG [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(1351): Waiting on c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:55,487 DEBUG [StoreFileCloser-c9c0b8ee4796868e2123ff071b738206-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/family/f8a8bae8822b4cf793c320938f3316b7 2024-11-19T21:54:55,487 WARN [hfile-prefetch-1732053291774 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/family/f8a8bae8822b4cf793c320938f3316b7, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-19T21:54:55,488 DEBUG [hfile-prefetch-1732053291774 {}] hfile.PrefetchExecutor(121): Prefetch completed for f8a8bae8822b4cf793c320938f3316b7 2024-11-19T21:54:55,491 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-19T21:54:55,491 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:55,491 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c9c0b8ee4796868e2123ff071b738206: Waiting for close lock at 1732053295486Running coprocessor pre-close hooks at 1732053295486Disabling compacts and flushes for region at 1732053295486Disabling writes for close at 1732053295486Writing region close event to WAL at 1732053295487 (+1 ms)Running coprocessor post-close hooks at 1732053295491 (+4 ms)Closed at 1732053295491 2024-11-19T21:54:55,491 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:55,516 INFO [regionserver/707262ffa86d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:54:55,518 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T21:54:55,518 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T21:54:55,518 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T21:54:55,518 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T21:54:55,523 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T21:54:55,523 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732053295518 2024-11-19T21:54:55,523 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T21:54:55,687 INFO [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(976): stopping server 707262ffa86d,46853,1732053292643; all regions closed. 2024-11-19T21:54:55,691 DEBUG [RS:1;707262ffa86d:46853 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs 2024-11-19T21:54:55,691 INFO [RS:1;707262ffa86d:46853 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 707262ffa86d%2C46853%2C1732053292643:(num 1732053293354) 2024-11-19T21:54:55,691 DEBUG [RS:1;707262ffa86d:46853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:55,691 INFO [RS:1;707262ffa86d:46853 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:54:55,691 INFO [RS:1;707262ffa86d:46853 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T21:54:55,692 INFO [RS:1;707262ffa86d:46853 {}] hbase.ChoreService(370): Chore service for: regionserver/707262ffa86d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T21:54:55,692 INFO [RS:1;707262ffa86d:46853 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T21:54:55,692 INFO [RS:1;707262ffa86d:46853 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T21:54:55,692 INFO [regionserver/707262ffa86d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T21:54:55,692 INFO [RS:1;707262ffa86d:46853 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T21:54:55,692 INFO [RS:1;707262ffa86d:46853 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T21:54:55,692 INFO [RS:1;707262ffa86d:46853 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46853 2024-11-19T21:54:55,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/707262ffa86d,46853,1732053292643 2024-11-19T21:54:55,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T21:54:55,721 INFO [RS:1;707262ffa86d:46853 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T21:54:55,730 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [707262ffa86d,46853,1732053292643] 2024-11-19T21:54:55,738 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/707262ffa86d,46853,1732053292643 already deleted, retry=false 2024-11-19T21:54:55,738 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 707262ffa86d,46853,1732053292643 on 707262ffa86d,41449,1732053292450 2024-11-19T21:54:55,746 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 707262ffa86d,46853,1732053292643, splitWal=true, meta=false 2024-11-19T21:54:55,749 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 707262ffa86d,46853,1732053292643 (carryingMeta=false) 707262ffa86d,46853,1732053292643/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@750cf83d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-19T21:54:55,749 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 707262ffa86d,46853,1732053292643, splitWal=true, meta=false 2024-11-19T21:54:55,751 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 707262ffa86d,46853,1732053292643 had 1 regions 2024-11-19T21:54:55,752 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 707262ffa86d,46853,1732053292643, splitWal=true, meta=false, isMeta: false 2024-11-19T21:54:55,753 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,46853,1732053292643-splitting 2024-11-19T21:54:55,753 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,46853,1732053292643-splitting dir is empty, no logs to split. 2024-11-19T21:54:55,753 INFO [PEWorker-2 {}] master.SplitWALManager(105): 707262ffa86d,46853,1732053292643 WAL count=0, meta=false 2024-11-19T21:54:55,755 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,46853,1732053292643-splitting dir is empty, no logs to split. 2024-11-19T21:54:55,755 INFO [PEWorker-2 {}] master.SplitWALManager(105): 707262ffa86d,46853,1732053292643 WAL count=0, meta=false 2024-11-19T21:54:55,755 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 707262ffa86d,46853,1732053292643 WAL splitting is done? wals=0, meta=false 2024-11-19T21:54:55,755 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 707262ffa86d,46853,1732053292643 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,46853,1732053292643-splitting does not exist 2024-11-19T21:54:55,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732053293784, region=c9c0b8ee4796868e2123ff071b738206, ASSIGN}] 2024-11-19T21:54:55,758 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732053293784, region=c9c0b8ee4796868e2123ff071b738206, ASSIGN 2024-11-19T21:54:55,758 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732053293784, region=c9c0b8ee4796868e2123ff071b738206, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T21:54:55,830 INFO [RS:1;707262ffa86d:46853 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T21:54:55,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:55,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46853-0x101566168560002, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:55,830 INFO [RS:1;707262ffa86d:46853 {}] regionserver.HRegionServer(1031): Exiting; stopping=707262ffa86d,46853,1732053292643; zookeeper connection closed. 2024-11-19T21:54:55,830 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7717e3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7717e3 2024-11-19T21:54:55,909 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c9c0b8ee4796868e2123ff071b738206, regionState=OPENING, regionLocation=707262ffa86d,36609,1732053292597 2024-11-19T21:54:55,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732053293784, region=c9c0b8ee4796868e2123ff071b738206, ASSIGN because future has completed 2024-11-19T21:54:55,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9c0b8ee4796868e2123ff071b738206, server=707262ffa86d,36609,1732053292597}] 2024-11-19T21:54:55,990 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/707262ffa86d:0 server-side Connection retries=45 2024-11-19T21:54:55,990 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:55,990 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:55,990 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T21:54:55,990 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:54:55,990 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T21:54:55,991 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T21:54:55,991 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T21:54:55,991 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36711 2024-11-19T21:54:55,993 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36711 connecting to ZooKeeper ensemble=127.0.0.1:57934 2024-11-19T21:54:55,993 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:55,993 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:56,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:367110x0, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T21:54:56,031 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36711-0x101566168560003 connected 2024-11-19T21:54:56,031 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36711-0x101566168560003, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-19T21:54:56,031 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-19T21:54:56,033 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T21:54:56,034 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.cache 2024-11-19T21:54:56,034 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T21:54:56,035 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T21:54:56,035 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T21:54:56,035 INFO [Thread-200 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T21:54:56,037 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T21:54:56,037 INFO [Thread-200 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-19T21:54:56,038 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T21:54:56,038 INFO [Thread-200 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-19T21:54:56,039 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36711-0x101566168560003, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T21:54:56,042 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36711-0x101566168560003, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T21:54:56,042 DEBUG [Thread-200 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-19T21:54:56,042 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36711 2024-11-19T21:54:56,042 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36711 2024-11-19T21:54:56,043 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T21:54:56,043 INFO [Thread-200 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-19T21:54:56,043 INFO [Thread-200 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.persistence is complete. 2024-11-19T21:54:56,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36711 2024-11-19T21:54:56,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36711 2024-11-19T21:54:56,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36711 2024-11-19T21:54:56,050 INFO [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer(746): ClusterId : 3ecd411f-96b3-4243-9240-1634950a407d 2024-11-19T21:54:56,050 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T21:54:56,056 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T21:54:56,056 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T21:54:56,064 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T21:54:56,064 DEBUG [RS:2;707262ffa86d:36711 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79432c11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=707262ffa86d/172.17.0.2:0 2024-11-19T21:54:56,066 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:56,066 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => c9c0b8ee4796868e2123ff071b738206, NAME => 'testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:54:56,067 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732053293784 c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,067 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:54:56,067 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,067 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,068 INFO [StoreOpener-c9c0b8ee4796868e2123ff071b738206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,068 INFO [StoreOpener-c9c0b8ee4796868e2123ff071b738206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9c0b8ee4796868e2123ff071b738206 columnFamilyName family 2024-11-19T21:54:56,068 DEBUG [StoreOpener-c9c0b8ee4796868e2123ff071b738206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:54:56,069 DEBUG [StoreFileOpener-c9c0b8ee4796868e2123ff071b738206-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/family/f8a8bae8822b4cf793c320938f3316b7, delay=968 ms 2024-11-19T21:54:56,070 DEBUG [StoreOpener-c9c0b8ee4796868e2123ff071b738206-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/family/f8a8bae8822b4cf793c320938f3316b7 2024-11-19T21:54:56,070 INFO [StoreOpener-c9c0b8ee4796868e2123ff071b738206-1 {}] regionserver.HStore(327): Store=c9c0b8ee4796868e2123ff071b738206/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:54:56,071 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,071 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,072 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,072 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,072 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,073 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,073 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened c9c0b8ee4796868e2123ff071b738206; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61563355, jitterRate=-0.08263452351093292}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T21:54:56,073 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,074 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for c9c0b8ee4796868e2123ff071b738206: Running coprocessor pre-open hook at 1732053296067Writing region info on filesystem at 1732053296067Initializing all the Stores at 1732053296067Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732053296067Cleaning up temporary data from old regions at 1732053296072 (+5 ms)Running coprocessor post-open hooks at 1732053296073 (+1 ms)Region opened successfully at 1732053296074 (+1 ms) 2024-11-19T21:54:56,074 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206., pid=9, masterSystemTime=1732053296064 2024-11-19T21:54:56,075 DEBUG [RS:2;707262ffa86d:36711 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;707262ffa86d:36711 2024-11-19T21:54:56,075 INFO [RS:2;707262ffa86d:36711 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T21:54:56,075 INFO [RS:2;707262ffa86d:36711 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T21:54:56,075 DEBUG [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T21:54:56,076 INFO [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer(2659): reportForDuty to master=707262ffa86d,41449,1732053292450 with port=36711, startcode=1732053295990 2024-11-19T21:54:56,077 DEBUG [RS:2;707262ffa86d:36711 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T21:54:56,077 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:56,077 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:56,078 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c9c0b8ee4796868e2123ff071b738206, regionState=OPEN, openSeqNum=10, regionLocation=707262ffa86d,36609,1732053292597 2024-11-19T21:54:56,079 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56197, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T21:54:56,079 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41449 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 707262ffa86d,36711,1732053295990 2024-11-19T21:54:56,079 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41449 {}] master.ServerManager(517): Registering regionserver=707262ffa86d,36711,1732053295990 2024-11-19T21:54:56,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9c0b8ee4796868e2123ff071b738206, server=707262ffa86d,36609,1732053292597 because future has completed 2024-11-19T21:54:56,080 DEBUG [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab 2024-11-19T21:54:56,080 DEBUG [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T21:54:56,081 DEBUG [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T21:54:56,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-19T21:54:56,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure c9c0b8ee4796868e2123ff071b738206, server=707262ffa86d,36609,1732053292597 in 170 msec 2024-11-19T21:54:56,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T21:54:56,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732053293784, region=c9c0b8ee4796868e2123ff071b738206, ASSIGN in 326 msec 2024-11-19T21:54:56,085 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 707262ffa86d,46853,1732053292643 after splitting done 2024-11-19T21:54:56,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 707262ffa86d,46853,1732053292643, splitWal=true, meta=false in 345 msec 2024-11-19T21:54:56,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T21:54:56,088 DEBUG [RS:2;707262ffa86d:36711 {}] zookeeper.ZKUtil(111): regionserver:36711-0x101566168560003, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/707262ffa86d,36711,1732053295990 2024-11-19T21:54:56,088 WARN [RS:2;707262ffa86d:36711 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T21:54:56,089 INFO [RS:2;707262ffa86d:36711 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:54:56,089 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [707262ffa86d,36711,1732053295990] 2024-11-19T21:54:56,089 DEBUG [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,36711,1732053295990 2024-11-19T21:54:56,093 INFO [RS:2;707262ffa86d:36711 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T21:54:56,095 INFO [RS:2;707262ffa86d:36711 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T21:54:56,096 INFO [RS:2;707262ffa86d:36711 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T21:54:56,096 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:56,096 INFO [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T21:54:56,097 INFO [RS:2;707262ffa86d:36711 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T21:54:56,097 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:56,097 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:56,097 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:56,097 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:56,097 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:56,097 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:56,097 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/707262ffa86d:0, corePoolSize=2, maxPoolSize=2 2024-11-19T21:54:56,097 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:56,098 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:56,098 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:56,098 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:56,098 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:56,098 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:54:56,098 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:54:56,098 DEBUG [RS:2;707262ffa86d:36711 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:54:56,099 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:56,099 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:56,099 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:56,099 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:56,099 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:56,099 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,36711,1732053295990-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T21:54:56,113 INFO [RS:2;707262ffa86d:36711 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T21:54:56,113 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,36711,1732053295990-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:56,113 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:56,113 INFO [RS:2;707262ffa86d:36711 {}] regionserver.Replication(171): 707262ffa86d,36711,1732053295990 started 2024-11-19T21:54:56,127 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:54:56,127 INFO [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer(1482): Serving as 707262ffa86d,36711,1732053295990, RpcServer on 707262ffa86d/172.17.0.2:36711, sessionid=0x101566168560003 2024-11-19T21:54:56,127 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T21:54:56,127 DEBUG [RS:2;707262ffa86d:36711 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 707262ffa86d,36711,1732053295990 2024-11-19T21:54:56,127 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,36711,1732053295990' 2024-11-19T21:54:56,127 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T21:54:56,128 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T21:54:56,128 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T21:54:56,128 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T21:54:56,128 DEBUG [RS:2;707262ffa86d:36711 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 707262ffa86d,36711,1732053295990 2024-11-19T21:54:56,129 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,36711,1732053295990' 2024-11-19T21:54:56,129 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T21:54:56,129 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T21:54:56,129 DEBUG [RS:2;707262ffa86d:36711 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T21:54:56,129 INFO [RS:2;707262ffa86d:36711 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T21:54:56,129 INFO [RS:2;707262ffa86d:36711 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T21:54:56,231 INFO [RS:2;707262ffa86d:36711 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T21:54:56,232 INFO [RS:2;707262ffa86d:36711 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=707262ffa86d%2C36711%2C1732053295990, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,36711,1732053295990, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs, maxLogs=55 2024-11-19T21:54:56,253 INFO [RS:2;707262ffa86d:36711 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/WALs/707262ffa86d,36711,1732053295990/707262ffa86d%2C36711%2C1732053295990.1732053296234 2024-11-19T21:54:56,253 DEBUG [RS:2;707262ffa86d:36711 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T21:54:56,508 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T21:54:56,509 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T21:54:56,511 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732053293784 2024-11-19T21:54:56,511 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732053293784 Metrics about Tables on a single HBase RegionServer 2024-11-19T21:54:56,513 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T21:54:56,513 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T21:54:56,513 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T21:54:56,513 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T21:54:56,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T21:54:56,627 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:54:56,627 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:54:56,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:56,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:56,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:54:56,628 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T21:54:56,628 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=152127592, stopped=false 2024-11-19T21:54:56,628 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=707262ffa86d,41449,1732053292450 2024-11-19T21:54:56,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36711-0x101566168560003, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:56,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:56,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T21:54:56,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:56,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:56,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36711-0x101566168560003, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:56,680 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T21:54:56,680 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:54:56,681 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:56,681 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36711-0x101566168560003, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:56,681 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:54:56,681 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:54:56,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:56,681 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '707262ffa86d,36609,1732053292597' ***** 2024-11-19T21:54:56,681 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T21:54:56,681 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '707262ffa86d,36711,1732053295990' ***** 2024-11-19T21:54:56,681 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T21:54:56,681 INFO [RS:0;707262ffa86d:36609 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T21:54:56,681 INFO [RS:2;707262ffa86d:36711 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T21:54:56,682 INFO [RS:0;707262ffa86d:36609 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.persistence 2024-11-19T21:54:56,682 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T21:54:56,682 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T21:54:56,682 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T21:54:56,682 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T21:54:56,682 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T21:54:56,682 INFO [RS:2;707262ffa86d:36711 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d9da2d86-61c4-4e85-1338-66cfaa86491d/bucket.persistence 2024-11-19T21:54:56,682 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T21:54:56,682 INFO [RS:0;707262ffa86d:36609 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T21:54:56,682 INFO [RS:2;707262ffa86d:36711 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T21:54:56,682 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T21:54:56,682 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T21:54:56,692 DEBUG [RS:0;707262ffa86d:36609 {}] bucket.BucketCache(1487): Thread RS:0;707262ffa86d:36609 started persisting bucket cache to file 2024-11-19T21:54:56,692 DEBUG [RS:2;707262ffa86d:36711 {}] bucket.BucketCache(1487): Thread RS:2;707262ffa86d:36711 started persisting bucket cache to file 2024-11-19T21:54:56,692 DEBUG [RS:0;707262ffa86d:36609 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T21:54:56,692 DEBUG [RS:2;707262ffa86d:36711 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T21:54:56,692 DEBUG [RS:0;707262ffa86d:36609 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T21:54:56,692 DEBUG [RS:2;707262ffa86d:36711 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T21:54:56,696 DEBUG [RS:2;707262ffa86d:36711 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T21:54:56,696 DEBUG [RS:0;707262ffa86d:36609 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T21:54:56,696 DEBUG [RS:0;707262ffa86d:36609 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732053296692 2024-11-19T21:54:56,696 DEBUG [RS:2;707262ffa86d:36711 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732053296692 2024-11-19T21:54:56,696 DEBUG [RS:2;707262ffa86d:36711 {}] bucket.BucketCache(1509): Thread RS:2;707262ffa86d:36711 finished persisting bucket cache to file, renaming 2024-11-19T21:54:56,696 INFO [RS:2;707262ffa86d:36711 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T21:54:56,696 INFO [RS:2;707262ffa86d:36711 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T21:54:56,696 INFO [RS:2;707262ffa86d:36711 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T21:54:56,696 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T21:54:56,696 INFO [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer(959): stopping server 707262ffa86d,36711,1732053295990 2024-11-19T21:54:56,696 INFO [RS:2;707262ffa86d:36711 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T21:54:56,697 INFO [RS:2;707262ffa86d:36711 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;707262ffa86d:36711. 2024-11-19T21:54:56,697 DEBUG [RS:2;707262ffa86d:36711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:54:56,697 DEBUG [RS:2;707262ffa86d:36711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:56,697 INFO [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer(976): stopping server 707262ffa86d,36711,1732053295990; all regions closed. 2024-11-19T21:54:56,699 DEBUG [RS:2;707262ffa86d:36711 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs 2024-11-19T21:54:56,699 INFO [RS:2;707262ffa86d:36711 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 707262ffa86d%2C36711%2C1732053295990:(num 1732053296234) 2024-11-19T21:54:56,699 DEBUG [RS:2;707262ffa86d:36711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:56,699 INFO [RS:2;707262ffa86d:36711 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:54:56,699 INFO [RS:2;707262ffa86d:36711 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T21:54:56,699 INFO [RS:2;707262ffa86d:36711 {}] hbase.ChoreService(370): Chore service for: regionserver/707262ffa86d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T21:54:56,699 INFO [RS:2;707262ffa86d:36711 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T21:54:56,699 INFO [RS:2;707262ffa86d:36711 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T21:54:56,699 INFO [regionserver/707262ffa86d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T21:54:56,699 INFO [RS:2;707262ffa86d:36711 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T21:54:56,699 INFO [RS:2;707262ffa86d:36711 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T21:54:56,699 INFO [RS:2;707262ffa86d:36711 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36711 2024-11-19T21:54:56,700 INFO [regionserver/707262ffa86d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:54:56,711 DEBUG [RS:0;707262ffa86d:36609 {}] bucket.BucketCache(1509): Thread RS:0;707262ffa86d:36609 finished persisting bucket cache to file, renaming 2024-11-19T21:54:56,711 WARN [RS:0;707262ffa86d:36609 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-19T21:54:56,711 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T21:54:56,711 INFO [RS:0;707262ffa86d:36609 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T21:54:56,711 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T21:54:56,711 INFO [RS:0;707262ffa86d:36609 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T21:54:56,711 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(3091): Received CLOSE for c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,712 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(959): stopping server 707262ffa86d,36609,1732053292597 2024-11-19T21:54:56,712 INFO [RS:0;707262ffa86d:36609 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T21:54:56,712 INFO [RS:0;707262ffa86d:36609 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;707262ffa86d:36609. 2024-11-19T21:54:56,712 DEBUG [RS:0;707262ffa86d:36609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:54:56,712 DEBUG [RS:0;707262ffa86d:36609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:56,712 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T21:54:56,712 INFO [RS:0;707262ffa86d:36609 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T21:54:56,712 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c9c0b8ee4796868e2123ff071b738206, disabling compactions & flushes 2024-11-19T21:54:56,712 INFO [RS:0;707262ffa86d:36609 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T21:54:56,712 INFO [RS:0;707262ffa86d:36609 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T21:54:56,712 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:56,712 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:56,712 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T21:54:56,712 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. after waiting 0 ms 2024-11-19T21:54:56,712 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:56,713 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-19T21:54:56,713 DEBUG [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(1325): Online Regions={c9c0b8ee4796868e2123ff071b738206=testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206., 1588230740=hbase:meta,,1.1588230740} 2024-11-19T21:54:56,713 DEBUG [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c9c0b8ee4796868e2123ff071b738206 2024-11-19T21:54:56,713 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T21:54:56,713 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T21:54:56,713 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T21:54:56,713 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T21:54:56,713 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T21:54:56,713 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T21:54:56,713 DEBUG [StoreFileCloser-c9c0b8ee4796868e2123ff071b738206-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/family/f8a8bae8822b4cf793c320938f3316b7 2024-11-19T21:54:56,713 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-19T21:54:56,718 INFO [regionserver/707262ffa86d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:54:56,718 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/default/testBlockEvictionOnGracefulStop1732053293784/c9c0b8ee4796868e2123ff071b738206/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-19T21:54:56,719 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:56,719 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c9c0b8ee4796868e2123ff071b738206: Waiting for close lock at 1732053296712Running coprocessor pre-close hooks at 1732053296712Disabling compacts and flushes for region at 1732053296712Disabling writes for close at 1732053296712Writing region close event to WAL at 1732053296713 (+1 ms)Running coprocessor post-close hooks at 1732053296719 (+6 ms)Closed at 1732053296719 2024-11-19T21:54:56,719 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206. 2024-11-19T21:54:56,725 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 834fa6edc98b4492a0c1801f3392f147 2024-11-19T21:54:56,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T21:54:56,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36711-0x101566168560003, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/707262ffa86d,36711,1732053295990 2024-11-19T21:54:56,730 INFO [RS:2;707262ffa86d:36711 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T21:54:56,738 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [707262ffa86d,36711,1732053295990] 2024-11-19T21:54:56,742 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/info/834fa6edc98b4492a0c1801f3392f147 is 209, key is testBlockEvictionOnGracefulStop1732053293784,,1732053293789.c9c0b8ee4796868e2123ff071b738206./info:regioninfo/1732053296078/Put/seqid=0 2024-11-19T21:54:56,743 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/info/834fa6edc98b4492a0c1801f3392f147 2024-11-19T21:54:56,746 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/707262ffa86d,36711,1732053295990 already deleted, retry=false 2024-11-19T21:54:56,746 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 707262ffa86d,36711,1732053295990 expired; onlineServers=1 2024-11-19T21:54:56,750 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d9a060cd5da44e11a2402bbafaaa5573 2024-11-19T21:54:56,763 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/ns/d9a060cd5da44e11a2402bbafaaa5573 is 43, key is default/ns:d/1732053293728/Put/seqid=0 2024-11-19T21:54:56,763 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/ns/d9a060cd5da44e11a2402bbafaaa5573 2024-11-19T21:54:56,771 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4683996c786c452eb0dbd0835580063e 2024-11-19T21:54:56,785 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/table/4683996c786c452eb0dbd0835580063e is 80, key is testBlockEvictionOnGracefulStop1732053293784/table:state/1732053294167/Put/seqid=0 2024-11-19T21:54:56,785 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/table/4683996c786c452eb0dbd0835580063e 2024-11-19T21:54:56,787 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/info/834fa6edc98b4492a0c1801f3392f147 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/info/834fa6edc98b4492a0c1801f3392f147 2024-11-19T21:54:56,789 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/info/834fa6edc98b4492a0c1801f3392f147, delay=942 ms 2024-11-19T21:54:56,790 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/info/834fa6edc98b4492a0c1801f3392f147, entries=14, sequenceid=13, filesize=7.5 K 2024-11-19T21:54:56,790 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/ns/d9a060cd5da44e11a2402bbafaaa5573 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/ns/d9a060cd5da44e11a2402bbafaaa5573 2024-11-19T21:54:56,791 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/ns/d9a060cd5da44e11a2402bbafaaa5573, delay=981 ms 2024-11-19T21:54:56,792 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/ns/d9a060cd5da44e11a2402bbafaaa5573, entries=2, sequenceid=13, filesize=5.0 K 2024-11-19T21:54:56,792 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/.tmp/table/4683996c786c452eb0dbd0835580063e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/table/4683996c786c452eb0dbd0835580063e 2024-11-19T21:54:56,793 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/table/4683996c786c452eb0dbd0835580063e, delay=917 ms 2024-11-19T21:54:56,794 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/table/4683996c786c452eb0dbd0835580063e, entries=2, sequenceid=13, filesize=5.3 K 2024-11-19T21:54:56,795 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=13, compaction requested=false 2024-11-19T21:54:56,795 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/info/834fa6edc98b4492a0c1801f3392f147 2024-11-19T21:54:56,796 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/ns/d9a060cd5da44e11a2402bbafaaa5573 2024-11-19T21:54:56,796 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/table/4683996c786c452eb0dbd0835580063e 2024-11-19T21:54:56,800 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-19T21:54:56,800 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T21:54:56,800 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T21:54:56,800 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732053296713Running coprocessor pre-close hooks at 1732053296713Disabling compacts and flushes for region at 1732053296713Disabling writes for close at 1732053296713Obtaining lock to block concurrent updates at 1732053296713Preparing flush snapshotting stores in 1588230740 at 1732053296713Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732053296714 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732053296714Flushing 1588230740/info: creating writer at 1732053296714Flushing 1588230740/info: appending metadata at 1732053296741 (+27 ms)Flushing 1588230740/info: closing flushed file at 1732053296741Flushing 1588230740/ns: creating writer at 1732053296745 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732053296762 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732053296762Flushing 1588230740/table: creating writer at 1732053296765 (+3 ms)Flushing 1588230740/table: appending metadata at 1732053296784 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732053296784Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69bd688b: reopening flushed file at 1732053296787 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e67fff9: reopening flushed file at 1732053296790 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ffcfe1c: reopening flushed file at 1732053296792 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=13, compaction requested=false at 1732053296795 (+3 ms)Writing region close event to WAL at 1732053296797 (+2 ms)Running coprocessor post-close hooks at 1732053296800 (+3 ms)Closed at 1732053296800 2024-11-19T21:54:56,801 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T21:54:56,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36711-0x101566168560003, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:56,838 INFO [RS:2;707262ffa86d:36711 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T21:54:56,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36711-0x101566168560003, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:56,838 INFO [RS:2;707262ffa86d:36711 {}] regionserver.HRegionServer(1031): Exiting; stopping=707262ffa86d,36711,1732053295990; zookeeper connection closed. 2024-11-19T21:54:56,838 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@50d01693 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@50d01693 2024-11-19T21:54:56,913 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(976): stopping server 707262ffa86d,36609,1732053292597; all regions closed. 2024-11-19T21:54:56,917 DEBUG [RS:0;707262ffa86d:36609 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs 2024-11-19T21:54:56,917 INFO [RS:0;707262ffa86d:36609 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 707262ffa86d%2C36609%2C1732053292597.meta:.meta(num 1732053293658) 2024-11-19T21:54:56,920 DEBUG [RS:0;707262ffa86d:36609 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/oldWALs 2024-11-19T21:54:56,920 INFO [RS:0;707262ffa86d:36609 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 707262ffa86d%2C36609%2C1732053292597:(num 1732053293351) 2024-11-19T21:54:56,920 DEBUG [RS:0;707262ffa86d:36609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:54:56,920 INFO [RS:0;707262ffa86d:36609 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:54:56,921 INFO [RS:0;707262ffa86d:36609 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T21:54:56,921 INFO [RS:0;707262ffa86d:36609 {}] hbase.ChoreService(370): Chore service for: regionserver/707262ffa86d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T21:54:56,921 INFO [RS:0;707262ffa86d:36609 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T21:54:56,922 INFO [regionserver/707262ffa86d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T21:54:56,922 INFO [RS:0;707262ffa86d:36609 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36609 2024-11-19T21:54:56,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/707262ffa86d,36609,1732053292597 2024-11-19T21:54:56,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T21:54:56,953 INFO [RS:0;707262ffa86d:36609 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T21:54:56,963 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [707262ffa86d,36609,1732053292597] 2024-11-19T21:54:56,971 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/707262ffa86d,36609,1732053292597 already deleted, retry=false 2024-11-19T21:54:56,971 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 707262ffa86d,36609,1732053292597 expired; onlineServers=0 2024-11-19T21:54:56,971 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '707262ffa86d,41449,1732053292450' ***** 2024-11-19T21:54:56,971 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T21:54:56,971 INFO [M:0;707262ffa86d:41449 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T21:54:56,971 INFO [M:0;707262ffa86d:41449 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T21:54:56,971 DEBUG [M:0;707262ffa86d:41449 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T21:54:56,971 DEBUG [M:0;707262ffa86d:41449 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T21:54:56,971 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T21:54:56,971 DEBUG [master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.small.0-1732053293003 {}] cleaner.HFileCleaner(306): Exit Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.small.0-1732053293003,5,FailOnTimeoutGroup] 2024-11-19T21:54:56,971 DEBUG [master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.large.0-1732053293003 {}] cleaner.HFileCleaner(306): Exit Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.large.0-1732053293003,5,FailOnTimeoutGroup] 2024-11-19T21:54:56,972 INFO [M:0;707262ffa86d:41449 {}] hbase.ChoreService(370): Chore service for: master/707262ffa86d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T21:54:56,972 INFO [M:0;707262ffa86d:41449 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T21:54:56,972 DEBUG [M:0;707262ffa86d:41449 {}] master.HMaster(1795): Stopping service threads 2024-11-19T21:54:56,972 INFO [M:0;707262ffa86d:41449 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T21:54:56,972 INFO [M:0;707262ffa86d:41449 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T21:54:56,972 INFO [M:0;707262ffa86d:41449 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T21:54:56,972 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T21:54:56,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T21:54:56,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:54:56,980 DEBUG [M:0;707262ffa86d:41449 {}] zookeeper.ZKUtil(347): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T21:54:56,980 WARN [M:0;707262ffa86d:41449 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T21:54:56,980 INFO [M:0;707262ffa86d:41449 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/.lastflushedseqids 2024-11-19T21:54:56,986 INFO [M:0;707262ffa86d:41449 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T21:54:56,986 INFO [M:0;707262ffa86d:41449 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T21:54:56,986 DEBUG [M:0;707262ffa86d:41449 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T21:54:56,986 INFO [M:0;707262ffa86d:41449 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:56,986 DEBUG [M:0;707262ffa86d:41449 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:56,986 DEBUG [M:0;707262ffa86d:41449 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T21:54:56,987 DEBUG [M:0;707262ffa86d:41449 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:56,987 INFO [M:0;707262ffa86d:41449 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-19T21:54:56,995 DEBUG [M:0;707262ffa86d:41449 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 06952b9f4f194e0f8d99c73d331bbf27 2024-11-19T21:54:57,006 DEBUG [M:0;707262ffa86d:41449 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/06952b9f4f194e0f8d99c73d331bbf27 is 82, key is hbase:meta,,1/info:regioninfo/1732053293690/Put/seqid=0 2024-11-19T21:54:57,006 INFO [M:0;707262ffa86d:41449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/06952b9f4f194e0f8d99c73d331bbf27 2024-11-19T21:54:57,013 DEBUG [M:0;707262ffa86d:41449 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1549fe816e6e4d20871969eb0abe9de6 2024-11-19T21:54:57,024 DEBUG [M:0;707262ffa86d:41449 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1549fe816e6e4d20871969eb0abe9de6 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732053294172/Put/seqid=0 2024-11-19T21:54:57,025 INFO [M:0;707262ffa86d:41449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1549fe816e6e4d20871969eb0abe9de6 2024-11-19T21:54:57,032 DEBUG [M:0;707262ffa86d:41449 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 56c0d5034ed34d51988cd8ab31b94318 2024-11-19T21:54:57,043 DEBUG [M:0;707262ffa86d:41449 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/56c0d5034ed34d51988cd8ab31b94318 is 69, key is 707262ffa86d,36609,1732053292597/rs:state/1732053293125/Put/seqid=0 2024-11-19T21:54:57,044 INFO [M:0;707262ffa86d:41449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/56c0d5034ed34d51988cd8ab31b94318 2024-11-19T21:54:57,045 INFO [M:0;707262ffa86d:41449 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56c0d5034ed34d51988cd8ab31b94318 2024-11-19T21:54:57,046 DEBUG [M:0;707262ffa86d:41449 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/06952b9f4f194e0f8d99c73d331bbf27 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/06952b9f4f194e0f8d99c73d331bbf27 2024-11-19T21:54:57,047 INFO [M:0;707262ffa86d:41449 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/06952b9f4f194e0f8d99c73d331bbf27, entries=8, sequenceid=83, filesize=5.5 K 2024-11-19T21:54:57,047 DEBUG [M:0;707262ffa86d:41449 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1549fe816e6e4d20871969eb0abe9de6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1549fe816e6e4d20871969eb0abe9de6 2024-11-19T21:54:57,049 INFO [M:0;707262ffa86d:41449 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1549fe816e6e4d20871969eb0abe9de6, entries=9, sequenceid=83, filesize=6.4 K 2024-11-19T21:54:57,049 DEBUG [M:0;707262ffa86d:41449 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/56c0d5034ed34d51988cd8ab31b94318 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/56c0d5034ed34d51988cd8ab31b94318 2024-11-19T21:54:57,050 INFO [M:0;707262ffa86d:41449 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56c0d5034ed34d51988cd8ab31b94318 2024-11-19T21:54:57,051 INFO [M:0;707262ffa86d:41449 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbd3bc59-dd95-49c4-e312-28ac5e068e9f/5785bdc0-9bf7-ddf8-22ec-178002676fab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/56c0d5034ed34d51988cd8ab31b94318, entries=3, sequenceid=83, filesize=5.3 K 2024-11-19T21:54:57,051 INFO [M:0;707262ffa86d:41449 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 64ms, sequenceid=83, compaction requested=false 2024-11-19T21:54:57,053 INFO [M:0;707262ffa86d:41449 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:54:57,053 DEBUG [M:0;707262ffa86d:41449 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732053296986Disabling compacts and flushes for region at 1732053296986Disabling writes for close at 1732053296987 (+1 ms)Obtaining lock to block concurrent updates at 1732053296987Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732053296987Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1732053296987Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732053296987Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732053296988 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732053297005 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732053297005Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732053297008 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732053297024 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732053297024Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732053297027 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732053297043 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732053297043Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d8a9eac: reopening flushed file at 1732053297045 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3408830f: reopening flushed file at 1732053297047 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30a99cca: reopening flushed file at 1732053297049 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 64ms, sequenceid=83, compaction requested=false at 1732053297051 (+2 ms)Writing region close event to WAL at 1732053297053 (+2 ms)Closed at 1732053297053 2024-11-19T21:54:57,054 INFO [M:0;707262ffa86d:41449 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T21:54:57,054 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T21:54:57,054 INFO [M:0;707262ffa86d:41449 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41449 2024-11-19T21:54:57,055 INFO [M:0;707262ffa86d:41449 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T21:54:57,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:57,063 INFO [RS:0;707262ffa86d:36609 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T21:54:57,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101566168560001, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:57,063 INFO [RS:0;707262ffa86d:36609 {}] regionserver.HRegionServer(1031): Exiting; stopping=707262ffa86d,36609,1732053292597; zookeeper connection closed. 2024-11-19T21:54:57,063 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@53cbb60f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@53cbb60f 2024-11-19T21:54:57,064 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-19T21:54:57,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:57,180 INFO [M:0;707262ffa86d:41449 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T21:54:57,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41449-0x101566168560000, quorum=127.0.0.1:57934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:54:57,184 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T21:54:57,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-19T21:54:57,186 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:54:57,193 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 50) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=378 (was 402), ProcessCount=11 (was 11), AvailableMemoryMB=5414 (was 5506)