2024-11-27 12:56:16,008 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-27 12:56:16,048 main DEBUG Took 0.037284 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 12:56:16,048 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 12:56:16,049 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 12:56:16,050 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 12:56:16,051 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,060 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 12:56:16,084 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,087 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,088 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,090 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,091 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,093 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,093 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,094 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,095 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,095 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,096 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,097 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,098 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,098 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,099 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,099 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,100 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:56:16,100 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,101 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 12:56:16,103 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:56:16,105 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 12:56:16,108 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 12:56:16,108 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 12:56:16,110 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 12:56:16,111 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 12:56:16,122 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 12:56:16,126 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 12:56:16,128 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 12:56:16,129 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 12:56:16,129 main DEBUG createAppenders(={Console}) 2024-11-27 12:56:16,130 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-27 12:56:16,131 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-27 12:56:16,131 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-27 12:56:16,132 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 12:56:16,132 main DEBUG OutputStream closed 2024-11-27 12:56:16,133 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 12:56:16,133 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 12:56:16,133 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-27 12:56:16,241 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 12:56:16,243 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 12:56:16,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 12:56:16,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 12:56:16,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 12:56:16,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 12:56:16,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 12:56:16,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 12:56:16,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 12:56:16,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 12:56:16,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 12:56:16,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 12:56:16,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 12:56:16,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 12:56:16,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 12:56:16,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 12:56:16,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 12:56:16,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 12:56:16,257 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 12:56:16,258 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-27 12:56:16,258 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 12:56:16,259 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-27T12:56:16,273 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-27 12:56:16,276 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 12:56:16,277 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T12:56:16,677 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c 2024-11-27T12:56:16,736 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=416, ProcessCount=11, AvailableMemoryMB=9475 2024-11-27T12:56:16,908 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T12:56:17,079 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:17,110 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/cluster_f697f418-0988-191b-b7d5-7e15adeaeb0a, deleteOnExit=true 2024-11-27T12:56:17,276 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/cluster_f697f418-0988-191b-b7d5-7e15adeaeb0a/zookeeper_0, clientPort=52061, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/cluster_f697f418-0988-191b-b7d5-7e15adeaeb0a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/cluster_f697f418-0988-191b-b7d5-7e15adeaeb0a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T12:56:17,299 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52061 2024-11-27T12:56:17,300 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:17,420 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6 with version=8 2024-11-27T12:56:17,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/hbase-staging 2024-11-27T12:56:17,558 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T12:56:17,923 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a3dd13ba50c2:0 server-side Connection retries=45 2024-11-27T12:56:17,933 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:17,935 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:17,946 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:56:17,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:17,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:56:18,113 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T12:56:18,197 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T12:56:18,238 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T12:56:18,251 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T12:56:18,256 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:56:18,302 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 108926 (auto-detected) 2024-11-27T12:56:18,304 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T12:56:18,322 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35619 2024-11-27T12:56:18,345 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35619 connecting to ZooKeeper ensemble=127.0.0.1:52061 2024-11-27T12:56:18,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:356190x0, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:56:18,460 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35619-0x1017da731190000 connected 2024-11-27T12:56:18,554 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:18,555 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:18,563 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:18,566 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6, hbase.cluster.distributed=false 2024-11-27T12:56:18,596 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:56:18,602 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35619 2024-11-27T12:56:18,602 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35619 2024-11-27T12:56:18,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35619 2024-11-27T12:56:18,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35619 2024-11-27T12:56:18,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35619 2024-11-27T12:56:18,710 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a3dd13ba50c2:0 server-side Connection retries=45 2024-11-27T12:56:18,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:18,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:18,713 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:56:18,713 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:18,713 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:56:18,717 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T12:56:18,721 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:56:18,722 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44891 2024-11-27T12:56:18,725 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44891 connecting to ZooKeeper ensemble=127.0.0.1:52061 2024-11-27T12:56:18,726 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:18,727 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:18,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448910x0, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:56:18,756 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:448910x0, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:18,756 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44891-0x1017da731190001 connected 2024-11-27T12:56:18,762 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T12:56:18,785 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/bucket.cache 2024-11-27T12:56:18,793 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-27T12:56:18,795 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-27T12:56:18,797 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-27T12:56:18,802 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-27T12:56:18,802 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-27T12:56:18,815 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-27T12:56:18,819 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-27T12:56:18,819 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/bucket.persistence is complete. 2024-11-27T12:56:18,827 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T12:56:18,830 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T12:56:18,840 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:56:18,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44891 2024-11-27T12:56:18,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44891 2024-11-27T12:56:18,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44891 2024-11-27T12:56:18,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44891 2024-11-27T12:56:18,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44891 2024-11-27T12:56:18,902 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a3dd13ba50c2:0 server-side Connection retries=45 2024-11-27T12:56:18,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:18,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:18,903 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:56:18,903 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:18,903 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:56:18,903 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T12:56:18,904 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:56:18,905 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36113 2024-11-27T12:56:18,909 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36113 connecting to ZooKeeper ensemble=127.0.0.1:52061 2024-11-27T12:56:18,910 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:18,911 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:18,948 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:361130x0, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:18,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361130x0, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:56:18,949 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T12:56:18,950 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36113-0x1017da731190002 connected 2024-11-27T12:56:18,951 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/bucket.cache 2024-11-27T12:56:18,951 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-27T12:56:18,952 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-27T12:56:18,952 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-27T12:56:18,953 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-27T12:56:18,953 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-27T12:56:18,957 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-27T12:56:18,958 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/bucket.persistence is complete. 2024-11-27T12:56:18,959 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-27T12:56:18,974 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T12:56:18,975 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T12:56:18,979 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:56:18,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36113 2024-11-27T12:56:18,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36113 2024-11-27T12:56:18,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36113 2024-11-27T12:56:19,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36113 2024-11-27T12:56:19,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36113 2024-11-27T12:56:19,026 DEBUG [M:0;a3dd13ba50c2:35619 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a3dd13ba50c2:35619 2024-11-27T12:56:19,027 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a3dd13ba50c2,35619,1732712177683 2024-11-27T12:56:19,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:19,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:19,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:19,080 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a3dd13ba50c2,35619,1732712177683 2024-11-27T12:56:19,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T12:56:19,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:19,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T12:56:19,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:19,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:19,123 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T12:56:19,125 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a3dd13ba50c2,35619,1732712177683 from backup master directory 2024-11-27T12:56:19,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:19,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a3dd13ba50c2,35619,1732712177683 2024-11-27T12:56:19,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:19,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:19,140 WARN [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:56:19,140 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a3dd13ba50c2,35619,1732712177683 2024-11-27T12:56:19,143 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T12:56:19,146 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T12:56:19,160 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/hbase.id] with ID: afbb6f7b-6707-4144-b2fa-41ef4480fb1f 2024-11-27T12:56:19,161 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/.tmp/hbase.id 2024-11-27T12:56:19,177 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/hbase.id] 2024-11-27T12:56:19,244 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:19,251 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T12:56:19,270 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-27T12:56:19,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:19,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:19,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:19,328 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:56:19,332 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T12:56:19,384 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:56:19,391 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:56:19,454 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store 2024-11-27T12:56:19,479 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T12:56:19,483 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:19,485 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T12:56:19,485 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:19,485 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:19,488 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T12:56:19,488 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:19,488 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:19,489 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732712179485Disabling compacts and flushes for region at 1732712179485Disabling writes for close at 1732712179488 (+3 ms)Writing region close event to WAL at 1732712179488Closed at 1732712179488 2024-11-27T12:56:19,490 WARN [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/.initializing 2024-11-27T12:56:19,490 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/WALs/a3dd13ba50c2,35619,1732712177683 2024-11-27T12:56:19,513 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:56:19,531 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3dd13ba50c2%2C35619%2C1732712177683, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/WALs/a3dd13ba50c2,35619,1732712177683, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/oldWALs, maxLogs=10 2024-11-27T12:56:19,579 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/WALs/a3dd13ba50c2,35619,1732712177683/a3dd13ba50c2%2C35619%2C1732712177683.1732712179537 2024-11-27T12:56:19,582 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T12:56:19,583 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:56:19,584 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:19,589 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,590 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T12:56:19,685 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:19,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:19,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,700 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T12:56:19,701 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:19,701 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:56:19,702 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,707 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T12:56:19,708 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:19,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:56:19,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T12:56:19,717 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:19,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:56:19,719 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,723 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,724 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,728 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,730 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,734 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:56:19,737 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:19,750 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:56:19,751 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58955070, jitterRate=-0.12150099873542786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:56:19,761 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732712179610Initializing all the Stores at 1732712179610Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712179611 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712179612 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712179613 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712179613Cleaning up temporary data from old regions at 1732712179730 (+117 ms)Region opened successfully at 1732712179761 (+31 ms) 2024-11-27T12:56:19,763 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T12:56:19,803 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ebde83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3dd13ba50c2/172.17.0.2:0 2024-11-27T12:56:19,846 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T12:56:19,862 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T12:56:19,862 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T12:56:19,866 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T12:56:19,867 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T12:56:19,875 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-27T12:56:19,875 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T12:56:19,913 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T12:56:19,925 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T12:56:19,984 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T12:56:19,987 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T12:56:19,989 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T12:56:20,000 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T12:56:20,003 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T12:56:20,008 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T12:56:20,018 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T12:56:20,020 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T12:56:20,029 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T12:56:20,054 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T12:56:20,063 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T12:56:20,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:20,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:20,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:20,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:20,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:20,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:20,096 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a3dd13ba50c2,35619,1732712177683, sessionid=0x1017da731190000, setting cluster-up flag (Was=false) 2024-11-27T12:56:20,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:20,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:20,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:20,176 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T12:56:20,178 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3dd13ba50c2,35619,1732712177683 2024-11-27T12:56:20,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:20,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:20,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:20,225 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T12:56:20,227 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3dd13ba50c2,35619,1732712177683 2024-11-27T12:56:20,235 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T12:56:20,310 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(746): ClusterId : afbb6f7b-6707-4144-b2fa-41ef4480fb1f 2024-11-27T12:56:20,311 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer(746): ClusterId : afbb6f7b-6707-4144-b2fa-41ef4480fb1f 2024-11-27T12:56:20,314 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T12:56:20,316 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T12:56:20,323 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T12:56:20,337 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T12:56:20,347 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T12:56:20,354 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T12:56:20,354 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T12:56:20,355 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T12:56:20,355 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T12:56:20,358 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a3dd13ba50c2,35619,1732712177683 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T12:56:20,374 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T12:56:20,374 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T12:56:20,375 DEBUG [RS:1;a3dd13ba50c2:36113 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4eb74e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3dd13ba50c2/172.17.0.2:0 2024-11-27T12:56:20,375 DEBUG [RS:0;a3dd13ba50c2:44891 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd76715, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3dd13ba50c2/172.17.0.2:0 2024-11-27T12:56:20,376 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a3dd13ba50c2:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:56:20,376 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a3dd13ba50c2:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:56:20,376 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a3dd13ba50c2:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:56:20,376 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a3dd13ba50c2:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:56:20,377 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a3dd13ba50c2:0, corePoolSize=10, maxPoolSize=10 2024-11-27T12:56:20,377 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,377 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a3dd13ba50c2:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:56:20,377 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,401 DEBUG [RS:0;a3dd13ba50c2:44891 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a3dd13ba50c2:44891 2024-11-27T12:56:20,406 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T12:56:20,406 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T12:56:20,406 DEBUG [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T12:56:20,411 DEBUG [RS:1;a3dd13ba50c2:36113 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a3dd13ba50c2:36113 2024-11-27T12:56:20,412 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T12:56:20,412 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T12:56:20,413 DEBUG [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T12:56:20,412 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(2659): reportForDuty to master=a3dd13ba50c2,35619,1732712177683 with port=44891, startcode=1732712178657 2024-11-27T12:56:20,414 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer(2659): reportForDuty to master=a3dd13ba50c2,35619,1732712177683 with port=36113, startcode=1732712178901 2024-11-27T12:56:20,423 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:56:20,424 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T12:56:20,429 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:20,430 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T12:56:20,437 DEBUG [RS:0;a3dd13ba50c2:44891 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T12:56:20,438 DEBUG [RS:1;a3dd13ba50c2:36113 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T12:56:20,448 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732712210448 2024-11-27T12:56:20,455 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T12:56:20,459 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T12:56:20,464 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T12:56:20,464 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T12:56:20,465 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T12:56:20,465 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T12:56:20,475 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,484 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T12:56:20,486 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T12:56:20,486 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T12:56:20,503 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T12:56:20,503 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T12:56:20,505 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T12:56:20,506 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6 2024-11-27T12:56:20,506 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.large.0-1732712180504,5,FailOnTimeoutGroup] 2024-11-27T12:56:20,510 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.small.0-1732712180506,5,FailOnTimeoutGroup] 2024-11-27T12:56:20,511 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,511 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T12:56:20,515 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,520 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:20,548 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36469, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T12:56:20,549 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37959, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T12:56:20,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-27T12:56:20,558 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35619 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a3dd13ba50c2,36113,1732712178901 2024-11-27T12:56:20,562 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35619 {}] master.ServerManager(517): Registering regionserver=a3dd13ba50c2,36113,1732712178901 2024-11-27T12:56:20,573 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T12:56:20,573 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:20,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:20,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-27T12:56:20,581 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35619 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:20,581 DEBUG [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6 2024-11-27T12:56:20,581 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35619 {}] master.ServerManager(517): Registering regionserver=a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:20,581 DEBUG [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-27T12:56:20,581 DEBUG [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T12:56:20,585 DEBUG [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6 2024-11-27T12:56:20,585 DEBUG [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-27T12:56:20,585 DEBUG [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T12:56:20,608 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T12:56:20,608 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:20,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:20,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-27T12:56:20,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:56:20,636 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T12:56:20,637 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:20,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:20,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-27T12:56:20,651 DEBUG [RS:1;a3dd13ba50c2:36113 {}] zookeeper.ZKUtil(111): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a3dd13ba50c2,36113,1732712178901 2024-11-27T12:56:20,652 WARN [RS:1;a3dd13ba50c2:36113 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:56:20,652 DEBUG [RS:0;a3dd13ba50c2:44891 {}] zookeeper.ZKUtil(111): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:20,652 INFO [RS:1;a3dd13ba50c2:36113 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:56:20,652 WARN [RS:0;a3dd13ba50c2:44891 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:56:20,653 DEBUG [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,36113,1732712178901 2024-11-27T12:56:20,653 INFO [RS:0;a3dd13ba50c2:44891 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:56:20,653 DEBUG [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:20,664 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T12:56:20,664 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:20,664 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a3dd13ba50c2,36113,1732712178901] 2024-11-27T12:56:20,665 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a3dd13ba50c2,44891,1732712178657] 2024-11-27T12:56:20,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:20,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T12:56:20,666 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740 2024-11-27T12:56:20,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740 2024-11-27T12:56:20,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T12:56:20,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T12:56:20,668 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:56:20,669 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T12:56:20,720 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T12:56:20,727 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:56:20,728 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64738135, jitterRate=-0.03532661497592926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:56:20,734 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T12:56:20,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732712180548Initializing all the Stores at 1732712180549 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712180549Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712180550 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712180550Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712180551 (+1 ms)Cleaning up temporary data from old regions at 1732712180667 (+116 ms)Region opened successfully at 1732712180736 (+69 ms) 2024-11-27T12:56:20,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T12:56:20,737 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T12:56:20,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T12:56:20,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T12:56:20,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T12:56:20,754 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T12:56:20,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732712180737Disabling compacts and flushes for region at 1732712180737Disabling writes for close at 1732712180737Writing region close event to WAL at 1732712180753 (+16 ms)Closed at 1732712180754 (+1 ms) 2024-11-27T12:56:20,757 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:56:20,757 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T12:56:20,767 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T12:56:20,771 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T12:56:20,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T12:56:20,777 INFO [RS:0;a3dd13ba50c2:44891 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T12:56:20,777 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,782 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:56:20,784 INFO [RS:1;a3dd13ba50c2:36113 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T12:56:20,785 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,787 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T12:56:20,798 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T12:56:20,798 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T12:56:20,804 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T12:56:20,806 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T12:56:20,811 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,812 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,812 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,812 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,814 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,815 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,815 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,815 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,815 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,815 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,816 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a3dd13ba50c2:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:56:20,816 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,816 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,816 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,816 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,817 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,817 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,817 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a3dd13ba50c2:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:56:20,817 DEBUG [RS:1;a3dd13ba50c2:36113 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a3dd13ba50c2:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:56:20,815 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,818 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,818 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a3dd13ba50c2:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:56:20,818 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,818 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,818 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,819 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,819 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,819 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:20,819 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a3dd13ba50c2:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:56:20,819 DEBUG [RS:0;a3dd13ba50c2:44891 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a3dd13ba50c2:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:56:20,838 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,839 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,839 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,839 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,839 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,840 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,44891,1732712178657-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:56:20,854 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,855 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,855 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,855 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,856 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,856 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,36113,1732712178901-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:56:20,879 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T12:56:20,882 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,44891,1732712178657-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,883 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,883 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.Replication(171): a3dd13ba50c2,44891,1732712178657 started 2024-11-27T12:56:20,893 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T12:56:20,893 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,36113,1732712178901-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,894 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,894 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.Replication(171): a3dd13ba50c2,36113,1732712178901 started 2024-11-27T12:56:20,914 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,915 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(1482): Serving as a3dd13ba50c2,44891,1732712178657, RpcServer on a3dd13ba50c2/172.17.0.2:44891, sessionid=0x1017da731190001 2024-11-27T12:56:20,916 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T12:56:20,916 DEBUG [RS:0;a3dd13ba50c2:44891 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:20,916 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3dd13ba50c2,44891,1732712178657' 2024-11-27T12:56:20,916 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T12:56:20,918 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T12:56:20,919 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T12:56:20,919 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T12:56:20,920 DEBUG [RS:0;a3dd13ba50c2:44891 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:20,920 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3dd13ba50c2,44891,1732712178657' 2024-11-27T12:56:20,920 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T12:56:20,921 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T12:56:20,922 DEBUG [RS:0;a3dd13ba50c2:44891 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T12:56:20,922 INFO [RS:0;a3dd13ba50c2:44891 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T12:56:20,922 INFO [RS:0;a3dd13ba50c2:44891 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T12:56:20,923 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:20,924 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer(1482): Serving as a3dd13ba50c2,36113,1732712178901, RpcServer on a3dd13ba50c2/172.17.0.2:36113, sessionid=0x1017da731190002 2024-11-27T12:56:20,924 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T12:56:20,924 DEBUG [RS:1;a3dd13ba50c2:36113 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a3dd13ba50c2,36113,1732712178901 2024-11-27T12:56:20,924 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3dd13ba50c2,36113,1732712178901' 2024-11-27T12:56:20,924 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T12:56:20,926 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T12:56:20,927 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T12:56:20,927 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T12:56:20,928 DEBUG [RS:1;a3dd13ba50c2:36113 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a3dd13ba50c2,36113,1732712178901 2024-11-27T12:56:20,928 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3dd13ba50c2,36113,1732712178901' 2024-11-27T12:56:20,928 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T12:56:20,932 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T12:56:20,933 DEBUG [RS:1;a3dd13ba50c2:36113 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T12:56:20,934 INFO [RS:1;a3dd13ba50c2:36113 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T12:56:20,934 INFO [RS:1;a3dd13ba50c2:36113 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T12:56:20,940 WARN [a3dd13ba50c2:35619 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T12:56:21,028 INFO [RS:0;a3dd13ba50c2:44891 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:56:21,029 INFO [RS:0;a3dd13ba50c2:44891 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a3dd13ba50c2%2C44891%2C1732712178657, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,44891,1732712178657, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs, maxLogs=55 2024-11-27T12:56:21,035 INFO [RS:1;a3dd13ba50c2:36113 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:56:21,036 INFO [RS:1;a3dd13ba50c2:36113 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a3dd13ba50c2%2C36113%2C1732712178901, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,36113,1732712178901, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs, maxLogs=55 2024-11-27T12:56:21,056 INFO [RS:0;a3dd13ba50c2:44891 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,44891,1732712178657/a3dd13ba50c2%2C44891%2C1732712178657.1732712181032 2024-11-27T12:56:21,058 DEBUG [RS:0;a3dd13ba50c2:44891 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T12:56:21,058 INFO [RS:1;a3dd13ba50c2:36113 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,36113,1732712178901/a3dd13ba50c2%2C36113%2C1732712178901.1732712181038 2024-11-27T12:56:21,060 DEBUG [RS:1;a3dd13ba50c2:36113 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T12:56:21,192 DEBUG [a3dd13ba50c2:35619 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-27T12:56:21,203 DEBUG [a3dd13ba50c2:35619 {}] balancer.BalancerClusterState(204): Hosts are {a3dd13ba50c2=0} racks are {/default-rack=0} 2024-11-27T12:56:21,206 DEBUG [a3dd13ba50c2:35619 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T12:56:21,206 DEBUG [a3dd13ba50c2:35619 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T12:56:21,206 DEBUG [a3dd13ba50c2:35619 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T12:56:21,206 DEBUG [a3dd13ba50c2:35619 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T12:56:21,206 INFO [a3dd13ba50c2:35619 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T12:56:21,206 INFO [a3dd13ba50c2:35619 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T12:56:21,206 DEBUG [a3dd13ba50c2:35619 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T12:56:21,214 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:21,220 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3dd13ba50c2,44891,1732712178657, state=OPENING 2024-11-27T12:56:21,267 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T12:56:21,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:21,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:21,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:21,314 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:21,314 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:21,315 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:21,317 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:56:21,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a3dd13ba50c2,44891,1732712178657}] 2024-11-27T12:56:21,491 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T12:56:21,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60771, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T12:56:21,503 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T12:56:21,504 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:56:21,505 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T12:56:21,507 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a3dd13ba50c2%2C44891%2C1732712178657.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,44891,1732712178657, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs, maxLogs=55 2024-11-27T12:56:21,531 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,44891,1732712178657/a3dd13ba50c2%2C44891%2C1732712178657.meta.1732712181510.meta 2024-11-27T12:56:21,531 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T12:56:21,531 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:56:21,533 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T12:56:21,535 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T12:56:21,540 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T12:56:21,544 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T12:56:21,544 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:21,545 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T12:56:21,545 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T12:56:21,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-27T12:56:21,548 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T12:56:21,548 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:21,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:21,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-27T12:56:21,549 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T12:56:21,549 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:21,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:21,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-27T12:56:21,550 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T12:56:21,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:21,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:21,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-27T12:56:21,551 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T12:56:21,551 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:21,552 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:21,552 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T12:56:21,554 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740 2024-11-27T12:56:21,557 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740 2024-11-27T12:56:21,558 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T12:56:21,558 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T12:56:21,558 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:56:21,560 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T12:56:21,561 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69716546, jitterRate=0.038857489824295044}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:56:21,561 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T12:56:21,563 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732712181545Writing region info on filesystem at 1732712181546 (+1 ms)Initializing all the Stores at 1732712181546Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712181546Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712181546Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712181546Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712181546Cleaning up temporary data from old regions at 1732712181558 (+12 ms)Running coprocessor post-open hooks at 1732712181561 (+3 ms)Region opened successfully at 1732712181563 (+2 ms) 2024-11-27T12:56:21,572 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732712181484 2024-11-27T12:56:21,586 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:21,586 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T12:56:21,587 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T12:56:21,588 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3dd13ba50c2,44891,1732712178657, state=OPEN 2024-11-27T12:56:21,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:56:21,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:56:21,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:56:21,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:21,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:21,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:21,606 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:21,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T12:56:21,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a3dd13ba50c2,44891,1732712178657 in 286 msec 2024-11-27T12:56:21,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T12:56:21,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 839 msec 2024-11-27T12:56:21,616 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:56:21,616 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T12:56:21,643 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:56:21,645 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3dd13ba50c2,44891,1732712178657, seqNum=-1] 2024-11-27T12:56:21,668 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:56:21,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50053, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:56:21,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4340 sec 2024-11-27T12:56:21,705 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732712181705, completionTime=-1 2024-11-27T12:56:21,708 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-27T12:56:21,709 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T12:56:21,740 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T12:56:21,741 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732712241741 2024-11-27T12:56:21,741 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732712301741 2024-11-27T12:56:21,741 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-27T12:56:21,744 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-27T12:56:21,755 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,35619,1732712177683-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:21,756 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,35619,1732712177683-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:21,756 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,35619,1732712177683-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:21,757 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a3dd13ba50c2:35619, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:21,758 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:21,762 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:21,770 DEBUG [master/a3dd13ba50c2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T12:56:21,805 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.664sec 2024-11-27T12:56:21,807 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T12:56:21,809 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T12:56:21,810 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T12:56:21,814 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T12:56:21,814 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T12:56:21,817 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,35619,1732712177683-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:56:21,819 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,35619,1732712177683-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T12:56:21,824 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T12:56:21,826 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T12:56:21,826 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,35619,1732712177683-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:21,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:56:21,855 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T12:56:21,855 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T12:56:21,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3dd13ba50c2,35619,-1 for getting cluster id 2024-11-27T12:56:21,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:56:21,906 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'afbb6f7b-6707-4144-b2fa-41ef4480fb1f' 2024-11-27T12:56:21,914 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:56:21,914 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "afbb6f7b-6707-4144-b2fa-41ef4480fb1f" 2024-11-27T12:56:21,915 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bdb3b30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:56:21,915 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3dd13ba50c2,35619,-1] 2024-11-27T12:56:21,931 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:56:21,934 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:21,944 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:56:21,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:56:21,952 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:56:21,963 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3dd13ba50c2,44891,1732712178657, seqNum=-1] 2024-11-27T12:56:21,964 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:56:21,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:56:22,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a3dd13ba50c2,35619,1732712177683 2024-11-27T12:56:22,015 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T12:56:22,021 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a3dd13ba50c2,35619,1732712177683 2024-11-27T12:56:22,024 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@18454fd1 2024-11-27T12:56:22,025 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T12:56:22,028 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T12:56:22,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732712182010', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:56:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35619 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732712182010 2024-11-27T12:56:22,046 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732712182010 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T12:56:22,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732712182010" procId is: 4 2024-11-27T12:56:22,049 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:22,052 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732712182010 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T12:56:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:56:22,089 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732712182010-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4e8facecf9ce7e75433ad4e0c14b8494, NAME => 'testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732712182010', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6 2024-11-27T12:56:22,097 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732712182010-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:22,097 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732712182010-pool-0 {}] regionserver.HRegion(1722): Closing 4e8facecf9ce7e75433ad4e0c14b8494, disabling compactions & flushes 2024-11-27T12:56:22,097 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732712182010-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:22,097 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732712182010-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:22,097 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732712182010-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. after waiting 0 ms 2024-11-27T12:56:22,097 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732712182010-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:22,097 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732712182010-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:22,098 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732712182010-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4e8facecf9ce7e75433ad4e0c14b8494: Waiting for close lock at 1732712182097Disabling compacts and flushes for region at 1732712182097Disabling writes for close at 1732712182097Writing region close event to WAL at 1732712182097Closed at 1732712182097 2024-11-27T12:56:22,101 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732712182010 execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T12:56:22,107 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732712182101"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732712182101"}]},"ts":"1732712182101"} 2024-11-27T12:56:22,116 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T12:56:22,123 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732712182010 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T12:56:22,127 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732712182010","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732712182123"}]},"ts":"1732712182123"} 2024-11-27T12:56:22,136 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732712182010, state=ENABLING in hbase:meta 2024-11-27T12:56:22,137 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a3dd13ba50c2=0} racks are {/default-rack=0} 2024-11-27T12:56:22,138 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T12:56:22,138 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T12:56:22,138 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T12:56:22,138 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T12:56:22,138 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T12:56:22,138 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T12:56:22,138 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T12:56:22,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732712182010, region=4e8facecf9ce7e75433ad4e0c14b8494, ASSIGN}] 2024-11-27T12:56:22,143 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732712182010, region=4e8facecf9ce7e75433ad4e0c14b8494, ASSIGN 2024-11-27T12:56:22,144 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732712182010, region=4e8facecf9ce7e75433ad4e0c14b8494, ASSIGN; state=OFFLINE, location=a3dd13ba50c2,36113,1732712178901; forceNewPlan=false, retain=false 2024-11-27T12:56:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:56:22,297 INFO [a3dd13ba50c2:35619 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T12:56:22,298 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4e8facecf9ce7e75433ad4e0c14b8494, regionState=OPENING, regionLocation=a3dd13ba50c2,36113,1732712178901 2024-11-27T12:56:22,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732712182010, region=4e8facecf9ce7e75433ad4e0c14b8494, ASSIGN because future has completed 2024-11-27T12:56:22,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8facecf9ce7e75433ad4e0c14b8494, server=a3dd13ba50c2,36113,1732712178901}] 2024-11-27T12:56:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:56:22,461 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T12:56:22,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41395, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T12:56:22,466 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:22,466 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e8facecf9ce7e75433ad4e0c14b8494, NAME => 'testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:56:22,467 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732712182010 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:22,467 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:22,467 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:22,467 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:22,469 INFO [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:22,475 INFO [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e8facecf9ce7e75433ad4e0c14b8494 columnFamilyName family 2024-11-27T12:56:22,475 DEBUG [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:22,476 INFO [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] regionserver.HStore(327): Store=4e8facecf9ce7e75433ad4e0c14b8494/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:56:22,476 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:22,477 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:22,477 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:22,477 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:22,477 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:22,479 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:22,488 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:56:22,489 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4e8facecf9ce7e75433ad4e0c14b8494; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67562311, jitterRate=0.006756886839866638}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:56:22,489 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:22,490 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4e8facecf9ce7e75433ad4e0c14b8494: Running coprocessor pre-open hook at 1732712182468Writing region info on filesystem at 1732712182468Initializing all the Stores at 1732712182468Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712182468Cleaning up temporary data from old regions at 1732712182478 (+10 ms)Running coprocessor post-open hooks at 1732712182489 (+11 ms)Region opened successfully at 1732712182490 (+1 ms) 2024-11-27T12:56:22,491 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494., pid=6, masterSystemTime=1732712182461 2024-11-27T12:56:22,495 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:22,495 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:22,496 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4e8facecf9ce7e75433ad4e0c14b8494, regionState=OPEN, openSeqNum=2, regionLocation=a3dd13ba50c2,36113,1732712178901 2024-11-27T12:56:22,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8facecf9ce7e75433ad4e0c14b8494, server=a3dd13ba50c2,36113,1732712178901 because future has completed 2024-11-27T12:56:22,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T12:56:22,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e8facecf9ce7e75433ad4e0c14b8494, server=a3dd13ba50c2,36113,1732712178901 in 196 msec 2024-11-27T12:56:22,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T12:56:22,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732712182010, region=4e8facecf9ce7e75433ad4e0c14b8494, ASSIGN in 363 msec 2024-11-27T12:56:22,507 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732712182010 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T12:56:22,507 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732712182010","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732712182507"}]},"ts":"1732712182507"} 2024-11-27T12:56:22,510 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732712182010, state=ENABLED in hbase:meta 2024-11-27T12:56:22,512 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732712182010 execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T12:56:22,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732712182010 in 473 msec 2024-11-27T12:56:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:56:22,697 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732712182010 get assigned. Timeout = 60000ms 2024-11-27T12:56:22,697 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732712182010 completed 2024-11-27T12:56:22,700 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T12:56:22,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732712182010 assigned to meta. Checking AM states. 2024-11-27T12:56:22,707 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T12:56:22,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732712182010 assigned. 2024-11-27T12:56:22,717 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732712182010', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494., hostname=a3dd13ba50c2,36113,1732712178901, seqNum=2] 2024-11-27T12:56:22,718 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:56:22,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:56:22,732 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4e8facecf9ce7e75433ad4e0c14b8494 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-27T12:56:22,761 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5b1b37d7600c495098597f03c2ee9dbb 2024-11-27T12:56:22,813 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/.tmp/family/5b1b37d7600c495098597f03c2ee9dbb is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-27T12:56:22,817 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/.tmp/family/5b1b37d7600c495098597f03c2ee9dbb 2024-11-27T12:56:22,850 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/.tmp/family/5b1b37d7600c495098597f03c2ee9dbb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb 2024-11-27T12:56:22,852 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb, delay=901 ms 2024-11-27T12:56:22,854 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb, entries=2, sequenceid=6, filesize=4.9 K 2024-11-27T12:56:22,861 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 4e8facecf9ce7e75433ad4e0c14b8494 in 128ms, sequenceid=6, compaction requested=false 2024-11-27T12:56:22,862 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732712182010' 2024-11-27T12:56:22,865 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4e8facecf9ce7e75433ad4e0c14b8494: 2024-11-27T12:56:23,756 DEBUG [hfile-prefetch-1732712182852 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5b1b37d7600c495098597f03c2ee9dbb_0. 2024-11-27T12:56:23,760 DEBUG [hfile-prefetch-1732712182852 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5b1b37d7600c495098597f03c2ee9dbb_117. 2024-11-27T12:56:23,761 DEBUG [hfile-prefetch-1732712182852 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb, with total blocks 2, and data blocks 1 2024-11-27T12:56:23,761 DEBUG [hfile-prefetch-1732712182852 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-27T12:56:23,762 DEBUG [hfile-prefetch-1732712182852 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-27T12:56:23,762 DEBUG [hfile-prefetch-1732712182852 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb. Will sleep 100ms and try the verification again. 2024-11-27T12:56:23,860 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-27T12:56:23,860 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-27T12:56:23,860 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T12:56:23,860 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T12:56:23,862 DEBUG [hfile-prefetch-1732712182852 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb, with total blocks 2, and data blocks 1 2024-11-27T12:56:23,862 DEBUG [hfile-prefetch-1732712182852 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-27T12:56:23,862 DEBUG [hfile-prefetch-1732712182852 {}] bucket.BucketCache(2293): found block for file 5b1b37d7600c495098597f03c2ee9dbb in the backing map. Acquiring read lock for offset 0 2024-11-27T12:56:23,863 DEBUG [hfile-prefetch-1732712182852 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb has now been fully cached. 2024-11-27T12:56:23,863 DEBUG [hfile-prefetch-1732712182852 {}] hfile.PrefetchExecutor(121): Prefetch completed for 5b1b37d7600c495098597f03c2ee9dbb 2024-11-27T12:56:23,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35619 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=4e8facecf9ce7e75433ad4e0c14b8494, source=a3dd13ba50c2,36113,1732712178901, destination=a3dd13ba50c2,44891,1732712178657, warming up region on a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:23,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35619 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=4e8facecf9ce7e75433ad4e0c14b8494, source=a3dd13ba50c2,36113,1732712178901, destination=a3dd13ba50c2,44891,1732712178657, running balancer 2024-11-27T12:56:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35619 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732712182010, region=4e8facecf9ce7e75433ad4e0c14b8494, REOPEN/MOVE 2024-11-27T12:56:23,882 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732712182010, region=4e8facecf9ce7e75433ad4e0c14b8494, REOPEN/MOVE 2024-11-27T12:56:23,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44891 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44891 {}] regionserver.HRegion(7855): Warmup {ENCODED => 4e8facecf9ce7e75433ad4e0c14b8494, NAME => 'testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:56:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44891 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:23,884 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4e8facecf9ce7e75433ad4e0c14b8494, regionState=CLOSING, regionLocation=a3dd13ba50c2,36113,1732712178901 2024-11-27T12:56:23,887 INFO [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:23,888 INFO [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e8facecf9ce7e75433ad4e0c14b8494 columnFamilyName family 2024-11-27T12:56:23,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732712182010, region=4e8facecf9ce7e75433ad4e0c14b8494, REOPEN/MOVE because future has completed 2024-11-27T12:56:23,888 DEBUG [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:23,889 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-27T12:56:23,890 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732712183860 2024-11-27T12:56:23,890 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-27T12:56:23,891 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-27T12:56:23,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e8facecf9ce7e75433ad4e0c14b8494, server=a3dd13ba50c2,36113,1732712178901}] 2024-11-27T12:56:23,896 DEBUG [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb 2024-11-27T12:56:23,896 INFO [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] regionserver.HStore(327): Store=4e8facecf9ce7e75433ad4e0c14b8494/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:56:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44891 {}] regionserver.HRegion(1722): Closing 4e8facecf9ce7e75433ad4e0c14b8494, disabling compactions & flushes 2024-11-27T12:56:23,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44891 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44891 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44891 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. after waiting 0 ms 2024-11-27T12:56:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44891 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:23,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44891 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44891 {}] regionserver.HRegion(1676): Region close journal for 4e8facecf9ce7e75433ad4e0c14b8494: Waiting for close lock at 1732712183896Disabling compacts and flushes for region at 1732712183896Disabling writes for close at 1732712183897 (+1 ms)Writing region close event to WAL at 1732712183910 (+13 ms)Closed at 1732712183910 2024-11-27T12:56:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35619 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-27T12:56:24,049 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,049 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-27T12:56:24,050 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 4e8facecf9ce7e75433ad4e0c14b8494, disabling compactions & flushes 2024-11-27T12:56:24,050 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,050 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,051 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. after waiting 0 ms 2024-11-27T12:56:24,051 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,053 DEBUG [StoreFileCloser-4e8facecf9ce7e75433ad4e0c14b8494-family-1 {}] bucket.BucketCache(768): Removing file 5b1b37d7600c495098597f03c2ee9dbb for region 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,053 DEBUG [StoreFileCloser-4e8facecf9ce7e75433ad4e0c14b8494-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 5b1b37d7600c495098597f03c2ee9dbb, starting offset: 0, end offset: 9223372036854775807 2024-11-27T12:56:24,055 DEBUG [StoreFileCloser-4e8facecf9ce7e75433ad4e0c14b8494-family-1 {}] bucket.BucketCache(856): removed key 5b1b37d7600c495098597f03c2ee9dbb_0 from back map with offset lock 2094080 in the evict process 2024-11-27T12:56:24,060 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-27T12:56:24,061 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,061 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 4e8facecf9ce7e75433ad4e0c14b8494: Waiting for close lock at 1732712184050Running coprocessor pre-close hooks at 1732712184050Disabling compacts and flushes for region at 1732712184050Disabling writes for close at 1732712184051 (+1 ms)Writing region close event to WAL at 1732712184055 (+4 ms)Running coprocessor post-close hooks at 1732712184060 (+5 ms)Closed at 1732712184061 (+1 ms) 2024-11-27T12:56:24,062 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 4e8facecf9ce7e75433ad4e0c14b8494 move to a3dd13ba50c2,44891,1732712178657 record at close sequenceid=6 2024-11-27T12:56:24,068 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,069 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4e8facecf9ce7e75433ad4e0c14b8494, regionState=CLOSED 2024-11-27T12:56:24,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e8facecf9ce7e75433ad4e0c14b8494, server=a3dd13ba50c2,36113,1732712178901 because future has completed 2024-11-27T12:56:24,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T12:56:24,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 4e8facecf9ce7e75433ad4e0c14b8494, server=a3dd13ba50c2,36113,1732712178901 in 182 msec 2024-11-27T12:56:24,076 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732712182010, region=4e8facecf9ce7e75433ad4e0c14b8494, REOPEN/MOVE; state=CLOSED, location=a3dd13ba50c2,44891,1732712178657; forceNewPlan=false, retain=false 2024-11-27T12:56:24,090 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-27T12:56:24,090 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-27T12:56:24,090 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T12:56:24,090 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T12:56:24,094 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T12:56:24,094 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732712184090 2024-11-27T12:56:24,094 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-27T12:56:24,226 INFO [a3dd13ba50c2:35619 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T12:56:24,227 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4e8facecf9ce7e75433ad4e0c14b8494, regionState=OPENING, regionLocation=a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:24,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732712182010, region=4e8facecf9ce7e75433ad4e0c14b8494, REOPEN/MOVE because future has completed 2024-11-27T12:56:24,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8facecf9ce7e75433ad4e0c14b8494, server=a3dd13ba50c2,44891,1732712178657}] 2024-11-27T12:56:24,386 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,387 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e8facecf9ce7e75433ad4e0c14b8494, NAME => 'testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:56:24,387 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732712182010 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,388 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:24,388 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,388 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,389 INFO [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,390 INFO [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e8facecf9ce7e75433ad4e0c14b8494 columnFamilyName family 2024-11-27T12:56:24,390 DEBUG [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:24,392 DEBUG [StoreFileOpener-4e8facecf9ce7e75433ad4e0c14b8494-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb, delay=933 ms 2024-11-27T12:56:24,394 DEBUG [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb 2024-11-27T12:56:24,394 INFO [StoreOpener-4e8facecf9ce7e75433ad4e0c14b8494-1 {}] regionserver.HStore(327): Store=4e8facecf9ce7e75433ad4e0c14b8494/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:56:24,394 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,395 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,396 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,396 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,396 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,402 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,403 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 4e8facecf9ce7e75433ad4e0c14b8494; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67931526, jitterRate=0.0122586190700531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:56:24,403 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,404 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 4e8facecf9ce7e75433ad4e0c14b8494: Running coprocessor pre-open hook at 1732712184388Writing region info on filesystem at 1732712184388Initializing all the Stores at 1732712184388Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712184389 (+1 ms)Cleaning up temporary data from old regions at 1732712184396 (+7 ms)Running coprocessor post-open hooks at 1732712184403 (+7 ms)Region opened successfully at 1732712184404 (+1 ms) 2024-11-27T12:56:24,405 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494., pid=9, masterSystemTime=1732712184383 2024-11-27T12:56:24,409 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4e8facecf9ce7e75433ad4e0c14b8494, regionState=OPEN, openSeqNum=10, regionLocation=a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:24,409 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,410 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8facecf9ce7e75433ad4e0c14b8494, server=a3dd13ba50c2,44891,1732712178657 because future has completed 2024-11-27T12:56:24,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-27T12:56:24,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e8facecf9ce7e75433ad4e0c14b8494, server=a3dd13ba50c2,44891,1732712178657 in 183 msec 2024-11-27T12:56:24,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732712182010, region=4e8facecf9ce7e75433ad4e0c14b8494, REOPEN/MOVE in 537 msec 2024-11-27T12:56:24,434 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T12:56:24,435 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:56:24,435 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:56:24,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:24,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:24,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:56:24,448 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T12:56:24,448 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-27T12:56:24,449 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a3dd13ba50c2,35619,1732712177683 2024-11-27T12:56:24,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:24,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:24,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:24,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:24,467 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T12:56:24,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:24,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:24,467 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:56:24,467 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:56:24,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:24,468 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a3dd13ba50c2,44891,1732712178657' ***** 2024-11-27T12:56:24,468 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T12:56:24,468 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a3dd13ba50c2,36113,1732712178901' ***** 2024-11-27T12:56:24,468 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T12:56:24,468 INFO [RS:0;a3dd13ba50c2:44891 {}] bucket.BucketCache(1738): Disabling cache 2024-11-27T12:56:24,468 INFO [RS:0;a3dd13ba50c2:44891 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/bucket.persistence 2024-11-27T12:56:24,469 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-27T12:56:24,469 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-27T12:56:24,469 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-27T12:56:24,469 INFO [RS:0;a3dd13ba50c2:44891 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-27T12:56:24,469 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-27T12:56:24,470 INFO [RS:1;a3dd13ba50c2:36113 {}] bucket.BucketCache(1738): Disabling cache 2024-11-27T12:56:24,470 INFO [RS:1;a3dd13ba50c2:36113 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/bucket.persistence 2024-11-27T12:56:24,471 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-27T12:56:24,471 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-27T12:56:24,471 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-27T12:56:24,471 INFO [RS:1;a3dd13ba50c2:36113 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-27T12:56:24,471 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:24,471 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-27T12:56:24,471 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:24,472 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:24,479 DEBUG [RS:0;a3dd13ba50c2:44891 {}] bucket.BucketCache(1487): Thread RS:0;a3dd13ba50c2:44891 started persisting bucket cache to file 2024-11-27T12:56:24,479 DEBUG [RS:0;a3dd13ba50c2:44891 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T12:56:24,479 DEBUG [RS:0;a3dd13ba50c2:44891 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T12:56:24,481 DEBUG [RS:1;a3dd13ba50c2:36113 {}] bucket.BucketCache(1487): Thread RS:1;a3dd13ba50c2:36113 started persisting bucket cache to file 2024-11-27T12:56:24,481 DEBUG [RS:1;a3dd13ba50c2:36113 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T12:56:24,481 DEBUG [RS:1;a3dd13ba50c2:36113 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T12:56:24,487 DEBUG [RS:0;a3dd13ba50c2:44891 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T12:56:24,487 DEBUG [RS:0;a3dd13ba50c2:44891 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732712184479 2024-11-27T12:56:24,487 DEBUG [RS:0;a3dd13ba50c2:44891 {}] bucket.BucketCache(1509): Thread RS:0;a3dd13ba50c2:44891 finished persisting bucket cache to file, renaming 2024-11-27T12:56:24,488 DEBUG [RS:1;a3dd13ba50c2:36113 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T12:56:24,488 DEBUG [RS:1;a3dd13ba50c2:36113 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732712184481 2024-11-27T12:56:24,488 DEBUG [RS:1;a3dd13ba50c2:36113 {}] bucket.BucketCache(1509): Thread RS:1;a3dd13ba50c2:36113 finished persisting bucket cache to file, renaming 2024-11-27T12:56:24,510 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T12:56:24,510 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T12:56:24,510 INFO [RS:0;a3dd13ba50c2:44891 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T12:56:24,510 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T12:56:24,510 INFO [RS:1;a3dd13ba50c2:36113 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T12:56:24,510 INFO [RS:0;a3dd13ba50c2:44891 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T12:56:24,510 INFO [RS:1;a3dd13ba50c2:36113 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T12:56:24,510 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer(959): stopping server a3dd13ba50c2,36113,1732712178901 2024-11-27T12:56:24,510 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(3091): Received CLOSE for 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,511 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:56:24,511 INFO [RS:1;a3dd13ba50c2:36113 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a3dd13ba50c2:36113. 2024-11-27T12:56:24,511 DEBUG [RS:1;a3dd13ba50c2:36113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:56:24,511 DEBUG [RS:1;a3dd13ba50c2:36113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:24,511 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(959): stopping server a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:24,511 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:56:24,511 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer(976): stopping server a3dd13ba50c2,36113,1732712178901; all regions closed. 2024-11-27T12:56:24,511 INFO [RS:0;a3dd13ba50c2:44891 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a3dd13ba50c2:44891. 2024-11-27T12:56:24,511 DEBUG [RS:0;a3dd13ba50c2:44891 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:56:24,511 DEBUG [RS:0;a3dd13ba50c2:44891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:24,512 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T12:56:24,512 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T12:56:24,512 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T12:56:24,512 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T12:56:24,512 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T12:56:24,513 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-27T12:56:24,514 DEBUG [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 4e8facecf9ce7e75433ad4e0c14b8494=testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494.} 2024-11-27T12:56:24,514 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-27T12:56:24,514 DEBUG [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4e8facecf9ce7e75433ad4e0c14b8494 2024-11-27T12:56:24,514 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-27T12:56:24,514 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4e8facecf9ce7e75433ad4e0c14b8494, disabling compactions & flushes 2024-11-27T12:56:24,514 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T12:56:24,514 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,514 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T12:56:24,515 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T12:56:24,515 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T12:56:24,515 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T12:56:24,517 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-27T12:56:24,518 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,518 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. after waiting 0 ms 2024-11-27T12:56:24,518 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,523 DEBUG [StoreFileCloser-4e8facecf9ce7e75433ad4e0c14b8494-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/family/5b1b37d7600c495098597f03c2ee9dbb 2024-11-27T12:56:24,533 DEBUG [RS:1;a3dd13ba50c2:36113 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs 2024-11-27T12:56:24,534 INFO [RS:1;a3dd13ba50c2:36113 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3dd13ba50c2%2C36113%2C1732712178901:(num 1732712181038) 2024-11-27T12:56:24,534 DEBUG [RS:1;a3dd13ba50c2:36113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:24,534 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:56:24,534 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:56:24,534 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.ChoreService(370): Chore service for: regionserver/a3dd13ba50c2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T12:56:24,535 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T12:56:24,535 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T12:56:24,535 INFO [regionserver/a3dd13ba50c2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:56:24,535 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T12:56:24,535 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:56:24,535 INFO [RS:1;a3dd13ba50c2:36113 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36113 2024-11-27T12:56:24,543 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnRegionMove1732712182010/4e8facecf9ce7e75433ad4e0c14b8494/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-27T12:56:24,543 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,544 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4e8facecf9ce7e75433ad4e0c14b8494: Waiting for close lock at 1732712184513Running coprocessor pre-close hooks at 1732712184514 (+1 ms)Disabling compacts and flushes for region at 1732712184514Disabling writes for close at 1732712184518 (+4 ms)Writing region close event to WAL at 1732712184523 (+5 ms)Running coprocessor post-close hooks at 1732712184543 (+20 ms)Closed at 1732712184543 2024-11-27T12:56:24,544 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494. 2024-11-27T12:56:24,549 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3d498a1734b5405ba15b66489d5902cf 2024-11-27T12:56:24,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:56:24,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a3dd13ba50c2,36113,1732712178901 2024-11-27T12:56:24,553 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:56:24,555 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a3dd13ba50c2,36113,1732712178901] 2024-11-27T12:56:24,557 INFO [regionserver/a3dd13ba50c2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:56:24,567 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a3dd13ba50c2,36113,1732712178901 already deleted, retry=false 2024-11-27T12:56:24,567 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a3dd13ba50c2,36113,1732712178901 expired; onlineServers=1 2024-11-27T12:56:24,568 INFO [regionserver/a3dd13ba50c2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:56:24,573 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/info/3d498a1734b5405ba15b66489d5902cf is 205, key is testBlockEvictionOnRegionMove1732712182010,,1732712182030.4e8facecf9ce7e75433ad4e0c14b8494./info:regioninfo/1732712184408/Put/seqid=0 2024-11-27T12:56:24,574 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/info/3d498a1734b5405ba15b66489d5902cf 2024-11-27T12:56:24,591 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2eefc720ded34a0495aa137e8a4de08c 2024-11-27T12:56:24,606 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/ns/2eefc720ded34a0495aa137e8a4de08c is 43, key is default/ns:d/1732712181678/Put/seqid=0 2024-11-27T12:56:24,607 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/ns/2eefc720ded34a0495aa137e8a4de08c 2024-11-27T12:56:24,618 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ed15d630c42a4a129a8e36a25dbe51c5 2024-11-27T12:56:24,635 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/table/ed15d630c42a4a129a8e36a25dbe51c5 is 78, key is testBlockEvictionOnRegionMove1732712182010/table:state/1732712182507/Put/seqid=0 2024-11-27T12:56:24,636 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/table/ed15d630c42a4a129a8e36a25dbe51c5 2024-11-27T12:56:24,639 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/info/3d498a1734b5405ba15b66489d5902cf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/3d498a1734b5405ba15b66489d5902cf 2024-11-27T12:56:24,640 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/3d498a1734b5405ba15b66489d5902cf, delay=972 ms 2024-11-27T12:56:24,642 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/3d498a1734b5405ba15b66489d5902cf, entries=15, sequenceid=15, filesize=7.6 K 2024-11-27T12:56:24,642 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/ns/2eefc720ded34a0495aa137e8a4de08c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/ns/2eefc720ded34a0495aa137e8a4de08c 2024-11-27T12:56:24,644 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/ns/2eefc720ded34a0495aa137e8a4de08c, delay=953 ms 2024-11-27T12:56:24,645 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/ns/2eefc720ded34a0495aa137e8a4de08c, entries=2, sequenceid=15, filesize=5.0 K 2024-11-27T12:56:24,646 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/table/ed15d630c42a4a129a8e36a25dbe51c5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/table/ed15d630c42a4a129a8e36a25dbe51c5 2024-11-27T12:56:24,647 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/table/ed15d630c42a4a129a8e36a25dbe51c5, delay=974 ms 2024-11-27T12:56:24,649 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/table/ed15d630c42a4a129a8e36a25dbe51c5, entries=2, sequenceid=15, filesize=5.3 K 2024-11-27T12:56:24,649 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 134ms, sequenceid=15, compaction requested=false 2024-11-27T12:56:24,650 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T12:56:24,651 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/3d498a1734b5405ba15b66489d5902cf 2024-11-27T12:56:24,654 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/ns/2eefc720ded34a0495aa137e8a4de08c 2024-11-27T12:56:24,655 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/table/ed15d630c42a4a129a8e36a25dbe51c5 2024-11-27T12:56:24,662 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-27T12:56:24,663 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T12:56:24,663 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T12:56:24,663 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732712184514Running coprocessor pre-close hooks at 1732712184514Disabling compacts and flushes for region at 1732712184514Disabling writes for close at 1732712184515 (+1 ms)Obtaining lock to block concurrent updates at 1732712184517 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1732712184517Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732712184518 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732712184518Flushing 1588230740/info: creating writer at 1732712184519 (+1 ms)Flushing 1588230740/info: appending metadata at 1732712184571 (+52 ms)Flushing 1588230740/info: closing flushed file at 1732712184572 (+1 ms)Flushing 1588230740/ns: creating writer at 1732712184578 (+6 ms)Flushing 1588230740/ns: appending metadata at 1732712184605 (+27 ms)Flushing 1588230740/ns: closing flushed file at 1732712184605Flushing 1588230740/table: creating writer at 1732712184610 (+5 ms)Flushing 1588230740/table: appending metadata at 1732712184634 (+24 ms)Flushing 1588230740/table: closing flushed file at 1732712184634Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e15fbea: reopening flushed file at 1732712184639 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cbd22c: reopening flushed file at 1732712184642 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2133402f: reopening flushed file at 1732712184645 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 134ms, sequenceid=15, compaction requested=false at 1732712184649 (+4 ms)Writing region close event to WAL at 1732712184655 (+6 ms)Running coprocessor post-close hooks at 1732712184662 (+7 ms)Closed at 1732712184663 (+1 ms) 2024-11-27T12:56:24,663 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T12:56:24,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:24,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36113-0x1017da731190002, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:24,668 INFO [RS:1;a3dd13ba50c2:36113 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:56:24,668 INFO [RS:1;a3dd13ba50c2:36113 {}] regionserver.HRegionServer(1031): Exiting; stopping=a3dd13ba50c2,36113,1732712178901; zookeeper connection closed. 2024-11-27T12:56:24,668 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2148992e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2148992e 2024-11-27T12:56:24,714 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(976): stopping server a3dd13ba50c2,44891,1732712178657; all regions closed. 2024-11-27T12:56:24,717 DEBUG [RS:0;a3dd13ba50c2:44891 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs 2024-11-27T12:56:24,717 INFO [RS:0;a3dd13ba50c2:44891 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3dd13ba50c2%2C44891%2C1732712178657.meta:.meta(num 1732712181510) 2024-11-27T12:56:24,719 DEBUG [RS:0;a3dd13ba50c2:44891 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs 2024-11-27T12:56:24,719 INFO [RS:0;a3dd13ba50c2:44891 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3dd13ba50c2%2C44891%2C1732712178657:(num 1732712181032) 2024-11-27T12:56:24,720 DEBUG [RS:0;a3dd13ba50c2:44891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:24,720 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:56:24,720 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:56:24,720 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.ChoreService(370): Chore service for: regionserver/a3dd13ba50c2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-27T12:56:24,720 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:56:24,720 INFO [regionserver/a3dd13ba50c2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:56:24,721 INFO [RS:0;a3dd13ba50c2:44891 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44891 2024-11-27T12:56:24,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a3dd13ba50c2,44891,1732712178657 2024-11-27T12:56:24,753 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:56:24,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:56:24,761 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a3dd13ba50c2,44891,1732712178657] 2024-11-27T12:56:24,770 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a3dd13ba50c2,44891,1732712178657 already deleted, retry=false 2024-11-27T12:56:24,770 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a3dd13ba50c2,44891,1732712178657 expired; onlineServers=0 2024-11-27T12:56:24,770 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a3dd13ba50c2,35619,1732712177683' ***** 2024-11-27T12:56:24,770 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T12:56:24,770 INFO [M:0;a3dd13ba50c2:35619 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:56:24,770 INFO [M:0;a3dd13ba50c2:35619 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:56:24,770 DEBUG [M:0;a3dd13ba50c2:35619 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T12:56:24,771 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T12:56:24,771 DEBUG [M:0;a3dd13ba50c2:35619 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T12:56:24,771 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.large.0-1732712180504 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.large.0-1732712180504,5,FailOnTimeoutGroup] 2024-11-27T12:56:24,771 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.small.0-1732712180506 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.small.0-1732712180506,5,FailOnTimeoutGroup] 2024-11-27T12:56:24,771 INFO [M:0;a3dd13ba50c2:35619 {}] hbase.ChoreService(370): Chore service for: master/a3dd13ba50c2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T12:56:24,771 INFO [M:0;a3dd13ba50c2:35619 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:56:24,771 DEBUG [M:0;a3dd13ba50c2:35619 {}] master.HMaster(1795): Stopping service threads 2024-11-27T12:56:24,771 INFO [M:0;a3dd13ba50c2:35619 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T12:56:24,772 INFO [M:0;a3dd13ba50c2:35619 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T12:56:24,772 INFO [M:0;a3dd13ba50c2:35619 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T12:56:24,772 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T12:56:24,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T12:56:24,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:24,779 DEBUG [M:0;a3dd13ba50c2:35619 {}] zookeeper.ZKUtil(347): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T12:56:24,779 WARN [M:0;a3dd13ba50c2:35619 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T12:56:24,779 INFO [M:0;a3dd13ba50c2:35619 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/.lastflushedseqids 2024-11-27T12:56:24,792 INFO [M:0;a3dd13ba50c2:35619 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T12:56:24,793 INFO [M:0;a3dd13ba50c2:35619 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T12:56:24,793 DEBUG [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T12:56:24,793 INFO [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:24,793 DEBUG [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:24,793 DEBUG [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T12:56:24,793 DEBUG [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:24,793 INFO [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-27T12:56:24,804 DEBUG [M:0;a3dd13ba50c2:35619 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1b67ed80d0f44b9ca023b515516c2948 2024-11-27T12:56:24,817 DEBUG [M:0;a3dd13ba50c2:35619 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b67ed80d0f44b9ca023b515516c2948 is 82, key is hbase:meta,,1/info:regioninfo/1732712181586/Put/seqid=0 2024-11-27T12:56:24,818 INFO [M:0;a3dd13ba50c2:35619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b67ed80d0f44b9ca023b515516c2948 2024-11-27T12:56:24,827 DEBUG [M:0;a3dd13ba50c2:35619 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 46dafd102d41484aa3f03f8a6ae47073 2024-11-27T12:56:24,841 DEBUG [M:0;a3dd13ba50c2:35619 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/46dafd102d41484aa3f03f8a6ae47073 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732712182513/Put/seqid=0 2024-11-27T12:56:24,841 INFO [M:0;a3dd13ba50c2:35619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/46dafd102d41484aa3f03f8a6ae47073 2024-11-27T12:56:24,850 DEBUG [M:0;a3dd13ba50c2:35619 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 58cf2c4dfdde471dac7887fb6ef71a6f 2024-11-27T12:56:24,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:24,862 INFO [RS:0;a3dd13ba50c2:44891 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:56:24,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44891-0x1017da731190001, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:24,862 INFO [RS:0;a3dd13ba50c2:44891 {}] regionserver.HRegionServer(1031): Exiting; stopping=a3dd13ba50c2,44891,1732712178657; zookeeper connection closed. 2024-11-27T12:56:24,862 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@799f1c6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@799f1c6 2024-11-27T12:56:24,863 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-27T12:56:24,863 DEBUG [M:0;a3dd13ba50c2:35619 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58cf2c4dfdde471dac7887fb6ef71a6f is 69, key is a3dd13ba50c2,36113,1732712178901/rs:state/1732712180565/Put/seqid=0 2024-11-27T12:56:24,863 INFO [M:0;a3dd13ba50c2:35619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58cf2c4dfdde471dac7887fb6ef71a6f 2024-11-27T12:56:24,865 DEBUG [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b67ed80d0f44b9ca023b515516c2948 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1b67ed80d0f44b9ca023b515516c2948 2024-11-27T12:56:24,868 INFO [M:0;a3dd13ba50c2:35619 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1b67ed80d0f44b9ca023b515516c2948, entries=8, sequenceid=84, filesize=5.5 K 2024-11-27T12:56:24,868 DEBUG [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/46dafd102d41484aa3f03f8a6ae47073 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/46dafd102d41484aa3f03f8a6ae47073 2024-11-27T12:56:24,876 INFO [M:0;a3dd13ba50c2:35619 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/46dafd102d41484aa3f03f8a6ae47073, entries=9, sequenceid=84, filesize=6.5 K 2024-11-27T12:56:24,877 DEBUG [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58cf2c4dfdde471dac7887fb6ef71a6f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/58cf2c4dfdde471dac7887fb6ef71a6f 2024-11-27T12:56:24,879 INFO [M:0;a3dd13ba50c2:35619 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/58cf2c4dfdde471dac7887fb6ef71a6f, entries=2, sequenceid=84, filesize=5.1 K 2024-11-27T12:56:24,880 INFO [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=84, compaction requested=false 2024-11-27T12:56:24,893 INFO [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:24,894 DEBUG [M:0;a3dd13ba50c2:35619 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732712184793Disabling compacts and flushes for region at 1732712184793Disabling writes for close at 1732712184793Obtaining lock to block concurrent updates at 1732712184793Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732712184793Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1732712184794 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732712184794Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732712184794Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732712184817 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732712184817Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732712184820 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732712184840 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732712184840Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732712184844 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732712184862 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732712184862Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ccfc570: reopening flushed file at 1732712184865 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fa73015: reopening flushed file at 1732712184868 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68bb6fc6: reopening flushed file at 1732712184876 (+8 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=84, compaction requested=false at 1732712184880 (+4 ms)Writing region close event to WAL at 1732712184893 (+13 ms)Closed at 1732712184893 2024-11-27T12:56:24,895 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:56:24,896 INFO [M:0;a3dd13ba50c2:35619 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T12:56:24,896 INFO [M:0;a3dd13ba50c2:35619 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35619 2024-11-27T12:56:24,896 INFO [M:0;a3dd13ba50c2:35619 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:56:25,012 INFO [M:0;a3dd13ba50c2:35619 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:56:25,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:25,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35619-0x1017da731190000, quorum=127.0.0.1:52061, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:25,017 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T12:56:25,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-27T12:56:25,043 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:25,053 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=52 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a3dd13ba50c2:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/a3dd13ba50c2:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732712184392 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/a3dd13ba50c2:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732712184644 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732712184641 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732712182852 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a3dd13ba50c2:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=467 (was 416) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9097 (was 9475) 2024-11-27T12:56:25,059 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=52, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=467, ProcessCount=11, AvailableMemoryMB=9096 2024-11-27T12:56:25,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/hadoop-log-dir so I do NOT create it in target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd 2024-11-27T12:56:25,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/hadoop-tmp-dir so I do NOT create it in target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd 2024-11-27T12:56:25,060 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:25,063 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/cluster_56e33791-33bd-a6a1-b39b-38ebc0192d27, deleteOnExit=true 2024-11-27T12:56:25,094 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/cluster_56e33791-33bd-a6a1-b39b-38ebc0192d27/zookeeper_0, clientPort=52261, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/cluster_56e33791-33bd-a6a1-b39b-38ebc0192d27/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/cluster_56e33791-33bd-a6a1-b39b-38ebc0192d27/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T12:56:25,095 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52261 2024-11-27T12:56:25,106 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6 with version=8 2024-11-27T12:56:25,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/hbase-staging 2024-11-27T12:56:25,108 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a3dd13ba50c2:0 server-side Connection retries=45 2024-11-27T12:56:25,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:25,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:25,109 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:56:25,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:25,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:56:25,109 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T12:56:25,109 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:56:25,110 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43841 2024-11-27T12:56:25,112 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43841 connecting to ZooKeeper ensemble=127.0.0.1:52261 2024-11-27T12:56:25,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:438410x0, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:56:25,153 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43841-0x1017da74fca0000 connected 2024-11-27T12:56:25,225 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:25,226 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:25,226 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:25,227 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6, hbase.cluster.distributed=false 2024-11-27T12:56:25,228 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:56:25,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43841 2024-11-27T12:56:25,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43841 2024-11-27T12:56:25,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43841 2024-11-27T12:56:25,235 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43841 2024-11-27T12:56:25,235 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43841 2024-11-27T12:56:25,256 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a3dd13ba50c2:0 server-side Connection retries=45 2024-11-27T12:56:25,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:25,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:25,256 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:56:25,257 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:25,257 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:56:25,257 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T12:56:25,257 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:56:25,258 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41341 2024-11-27T12:56:25,260 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41341 connecting to ZooKeeper ensemble=127.0.0.1:52261 2024-11-27T12:56:25,261 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:25,261 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:25,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413410x0, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:56:25,273 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:413410x0, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:25,273 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41341-0x1017da74fca0001 connected 2024-11-27T12:56:25,273 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T12:56:25,274 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.cache 2024-11-27T12:56:25,275 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-27T12:56:25,275 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-27T12:56:25,275 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-27T12:56:25,277 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-27T12:56:25,277 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-27T12:56:25,278 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-27T12:56:25,278 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.persistence is complete. 2024-11-27T12:56:25,279 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-27T12:56:25,282 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T12:56:25,283 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T12:56:25,285 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:56:25,298 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41341 2024-11-27T12:56:25,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41341 2024-11-27T12:56:25,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41341 2024-11-27T12:56:25,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41341 2024-11-27T12:56:25,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41341 2024-11-27T12:56:25,326 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a3dd13ba50c2:0 server-side Connection retries=45 2024-11-27T12:56:25,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:25,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:25,326 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:56:25,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:25,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:56:25,327 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T12:56:25,327 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:56:25,328 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40427 2024-11-27T12:56:25,330 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40427 connecting to ZooKeeper ensemble=127.0.0.1:52261 2024-11-27T12:56:25,330 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:25,331 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:25,354 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404270x0, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:56:25,355 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:404270x0, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:25,355 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T12:56:25,364 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40427-0x1017da74fca0002 connected 2024-11-27T12:56:25,367 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.cache 2024-11-27T12:56:25,367 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-27T12:56:25,367 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-27T12:56:25,368 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-27T12:56:25,381 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-27T12:56:25,381 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-27T12:56:25,381 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-27T12:56:25,382 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-27T12:56:25,382 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.persistence is complete. 2024-11-27T12:56:25,382 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T12:56:25,383 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T12:56:25,384 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:56:25,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40427 2024-11-27T12:56:25,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40427 2024-11-27T12:56:25,394 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40427 2024-11-27T12:56:25,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40427 2024-11-27T12:56:25,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40427 2024-11-27T12:56:25,412 DEBUG [M:0;a3dd13ba50c2:43841 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a3dd13ba50c2:43841 2024-11-27T12:56:25,414 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a3dd13ba50c2,43841,1732712185108 2024-11-27T12:56:25,431 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:25,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:25,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:25,432 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a3dd13ba50c2,43841,1732712185108 2024-11-27T12:56:25,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T12:56:25,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,442 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T12:56:25,442 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,443 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T12:56:25,444 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a3dd13ba50c2,43841,1732712185108 from backup master directory 2024-11-27T12:56:25,451 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:25,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:25,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a3dd13ba50c2,43841,1732712185108 2024-11-27T12:56:25,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:56:25,451 WARN [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:56:25,451 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a3dd13ba50c2,43841,1732712185108 2024-11-27T12:56:25,453 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/hbase.id] with ID: 49a81152-fef5-4818-bb52-858e1a5a1942 2024-11-27T12:56:25,453 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/.tmp/hbase.id 2024-11-27T12:56:25,460 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/hbase.id] 2024-11-27T12:56:25,509 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:25,509 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T12:56:25,512 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-27T12:56:25,547 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,552 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:56:25,553 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T12:56:25,554 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:56:25,570 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store 2024-11-27T12:56:25,576 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:25,576 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T12:56:25,577 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:25,577 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:25,577 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T12:56:25,577 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:25,577 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:25,577 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732712185576Disabling compacts and flushes for region at 1732712185576Disabling writes for close at 1732712185577 (+1 ms)Writing region close event to WAL at 1732712185577Closed at 1732712185577 2024-11-27T12:56:25,577 WARN [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/.initializing 2024-11-27T12:56:25,577 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/WALs/a3dd13ba50c2,43841,1732712185108 2024-11-27T12:56:25,583 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:56:25,585 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3dd13ba50c2%2C43841%2C1732712185108, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/WALs/a3dd13ba50c2,43841,1732712185108, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/oldWALs, maxLogs=10 2024-11-27T12:56:25,608 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/WALs/a3dd13ba50c2,43841,1732712185108/a3dd13ba50c2%2C43841%2C1732712185108.1732712185586 2024-11-27T12:56:25,608 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T12:56:25,608 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:56:25,608 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:25,609 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,609 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,610 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T12:56:25,615 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:25,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:25,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T12:56:25,624 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:25,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:56:25,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T12:56:25,628 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:25,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:56:25,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T12:56:25,632 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:25,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:56:25,633 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,633 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,634 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,634 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,634 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,634 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:56:25,635 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:56:25,641 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:56:25,641 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74270720, jitterRate=0.106719970703125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:56:25,642 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732712185609Initializing all the Stores at 1732712185609Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712185609Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712185610 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712185610Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712185610Cleaning up temporary data from old regions at 1732712185634 (+24 ms)Region opened successfully at 1732712185642 (+8 ms) 2024-11-27T12:56:25,643 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T12:56:25,647 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cffa2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3dd13ba50c2/172.17.0.2:0 2024-11-27T12:56:25,648 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T12:56:25,648 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T12:56:25,648 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T12:56:25,648 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T12:56:25,649 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T12:56:25,649 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-27T12:56:25,649 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T12:56:25,651 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T12:56:25,652 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T12:56:25,689 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T12:56:25,689 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T12:56:25,690 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T12:56:25,700 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T12:56:25,700 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T12:56:25,701 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T12:56:25,708 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T12:56:25,709 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T12:56:25,717 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T12:56:25,719 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T12:56:25,730 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T12:56:25,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:25,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:25,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,742 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:25,742 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,743 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a3dd13ba50c2,43841,1732712185108, sessionid=0x1017da74fca0000, setting cluster-up flag (Was=false) 2024-11-27T12:56:25,758 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,783 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T12:56:25,785 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3dd13ba50c2,43841,1732712185108 2024-11-27T12:56:25,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,800 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:25,825 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T12:56:25,827 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3dd13ba50c2,43841,1732712185108 2024-11-27T12:56:25,827 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T12:56:25,829 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T12:56:25,829 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T12:56:25,830 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T12:56:25,830 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a3dd13ba50c2,43841,1732712185108 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T12:56:25,832 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a3dd13ba50c2:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:56:25,832 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a3dd13ba50c2:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:56:25,832 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a3dd13ba50c2:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:56:25,832 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a3dd13ba50c2:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:56:25,832 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a3dd13ba50c2:0, corePoolSize=10, maxPoolSize=10 2024-11-27T12:56:25,832 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:25,833 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a3dd13ba50c2:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:56:25,833 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:25,836 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732712215836 2024-11-27T12:56:25,836 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T12:56:25,836 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T12:56:25,836 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T12:56:25,836 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T12:56:25,836 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T12:56:25,836 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T12:56:25,838 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:25,839 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T12:56:25,839 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:56:25,839 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T12:56:25,839 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T12:56:25,839 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T12:56:25,840 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:25,840 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T12:56:25,842 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T12:56:25,843 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T12:56:25,843 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.large.0-1732712185843,5,FailOnTimeoutGroup] 2024-11-27T12:56:25,846 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.small.0-1732712185843,5,FailOnTimeoutGroup] 2024-11-27T12:56:25,846 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:25,847 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T12:56:25,847 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:25,847 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:25,865 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T12:56:25,865 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6 2024-11-27T12:56:25,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:25,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-27T12:56:25,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T12:56:25,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:25,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:25,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-27T12:56:25,895 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T12:56:25,895 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:25,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:25,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-27T12:56:25,902 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(746): ClusterId : 49a81152-fef5-4818-bb52-858e1a5a1942 2024-11-27T12:56:25,902 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T12:56:25,902 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(746): ClusterId : 49a81152-fef5-4818-bb52-858e1a5a1942 2024-11-27T12:56:25,902 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T12:56:25,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T12:56:25,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:25,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:25,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-27T12:56:25,915 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T12:56:25,915 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:25,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:25,916 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T12:56:25,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740 2024-11-27T12:56:25,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740 2024-11-27T12:56:25,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T12:56:25,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T12:56:25,918 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:56:25,918 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T12:56:25,929 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:56:25,929 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74117128, jitterRate=0.10443127155303955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:56:25,930 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732712185879Initializing all the Stores at 1732712185879Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712185879Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712185879Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712185879Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712185879Cleaning up temporary data from old regions at 1732712185917 (+38 ms)Region opened successfully at 1732712185930 (+13 ms) 2024-11-27T12:56:25,930 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T12:56:25,930 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T12:56:25,931 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T12:56:25,931 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T12:56:25,931 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T12:56:25,931 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T12:56:25,931 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732712185930Disabling compacts and flushes for region at 1732712185930Disabling writes for close at 1732712185931 (+1 ms)Writing region close event to WAL at 1732712185931Closed at 1732712185931 2024-11-27T12:56:25,932 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T12:56:25,932 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T12:56:25,932 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T12:56:25,932 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T12:56:25,932 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:56:25,933 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T12:56:25,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T12:56:25,934 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:56:25,935 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T12:56:25,943 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T12:56:25,943 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T12:56:25,943 DEBUG [RS:0;a3dd13ba50c2:41341 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2140af49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3dd13ba50c2/172.17.0.2:0 2024-11-27T12:56:25,943 DEBUG [RS:1;a3dd13ba50c2:40427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a42ffa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3dd13ba50c2/172.17.0.2:0 2024-11-27T12:56:25,959 DEBUG [RS:0;a3dd13ba50c2:41341 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a3dd13ba50c2:41341 2024-11-27T12:56:25,959 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T12:56:25,959 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T12:56:25,959 DEBUG [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T12:56:25,960 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(2659): reportForDuty to master=a3dd13ba50c2,43841,1732712185108 with port=41341, startcode=1732712185256 2024-11-27T12:56:25,961 DEBUG [RS:0;a3dd13ba50c2:41341 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T12:56:25,961 DEBUG [RS:1;a3dd13ba50c2:40427 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a3dd13ba50c2:40427 2024-11-27T12:56:25,962 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T12:56:25,962 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T12:56:25,962 DEBUG [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T12:56:25,964 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(2659): reportForDuty to master=a3dd13ba50c2,43841,1732712185108 with port=40427, startcode=1732712185325 2024-11-27T12:56:25,964 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47725, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T12:56:25,964 DEBUG [RS:1;a3dd13ba50c2:40427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T12:56:25,965 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43841 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:25,965 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43841 {}] master.ServerManager(517): Registering regionserver=a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:25,971 DEBUG [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6 2024-11-27T12:56:25,971 DEBUG [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-27T12:56:25,971 DEBUG [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T12:56:25,971 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42653, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T12:56:25,972 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43841 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:25,972 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43841 {}] master.ServerManager(517): Registering regionserver=a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:25,974 DEBUG [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6 2024-11-27T12:56:25,974 DEBUG [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-27T12:56:25,974 DEBUG [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T12:56:25,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:56:26,013 DEBUG [RS:0;a3dd13ba50c2:41341 {}] zookeeper.ZKUtil(111): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:26,013 WARN [RS:0;a3dd13ba50c2:41341 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:56:26,013 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a3dd13ba50c2,40427,1732712185325] 2024-11-27T12:56:26,014 INFO [RS:0;a3dd13ba50c2:41341 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:56:26,014 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a3dd13ba50c2,41341,1732712185256] 2024-11-27T12:56:26,014 DEBUG [RS:1;a3dd13ba50c2:40427 {}] zookeeper.ZKUtil(111): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:26,014 DEBUG [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:26,014 WARN [RS:1;a3dd13ba50c2:40427 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:56:26,014 INFO [RS:1;a3dd13ba50c2:40427 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:56:26,014 DEBUG [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:26,020 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T12:56:26,020 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T12:56:26,022 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T12:56:26,023 INFO [RS:0;a3dd13ba50c2:41341 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T12:56:26,023 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,023 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T12:56:26,024 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T12:56:26,024 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T12:56:26,024 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,024 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,024 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,024 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,024 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,024 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,024 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a3dd13ba50c2:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:56:26,025 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,025 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,025 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,025 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,025 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,025 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,025 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a3dd13ba50c2:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:56:26,025 DEBUG [RS:0;a3dd13ba50c2:41341 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a3dd13ba50c2:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:56:26,025 INFO [RS:1;a3dd13ba50c2:40427 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T12:56:26,025 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,026 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,026 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,026 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,027 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,027 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,027 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,41341,1732712185256-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:56:26,031 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T12:56:26,032 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T12:56:26,032 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,032 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,032 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,032 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,032 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,032 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,033 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a3dd13ba50c2:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:56:26,033 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,033 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,033 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,033 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,033 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,033 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:26,033 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a3dd13ba50c2:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:56:26,033 DEBUG [RS:1;a3dd13ba50c2:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a3dd13ba50c2:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:56:26,034 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,034 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,035 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,035 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,035 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,035 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,40427,1732712185325-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:56:26,050 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T12:56:26,050 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,41341,1732712185256-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,051 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,051 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.Replication(171): a3dd13ba50c2,41341,1732712185256 started 2024-11-27T12:56:26,059 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T12:56:26,059 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,40427,1732712185325-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,059 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,059 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.Replication(171): a3dd13ba50c2,40427,1732712185325 started 2024-11-27T12:56:26,070 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,070 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(1482): Serving as a3dd13ba50c2,41341,1732712185256, RpcServer on a3dd13ba50c2/172.17.0.2:41341, sessionid=0x1017da74fca0001 2024-11-27T12:56:26,070 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T12:56:26,071 DEBUG [RS:0;a3dd13ba50c2:41341 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:26,071 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3dd13ba50c2,41341,1732712185256' 2024-11-27T12:56:26,071 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T12:56:26,071 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T12:56:26,072 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T12:56:26,072 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T12:56:26,072 DEBUG [RS:0;a3dd13ba50c2:41341 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:26,072 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3dd13ba50c2,41341,1732712185256' 2024-11-27T12:56:26,072 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T12:56:26,073 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T12:56:26,073 DEBUG [RS:0;a3dd13ba50c2:41341 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T12:56:26,073 INFO [RS:0;a3dd13ba50c2:41341 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T12:56:26,073 INFO [RS:0;a3dd13ba50c2:41341 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T12:56:26,077 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,077 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(1482): Serving as a3dd13ba50c2,40427,1732712185325, RpcServer on a3dd13ba50c2/172.17.0.2:40427, sessionid=0x1017da74fca0002 2024-11-27T12:56:26,077 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T12:56:26,077 DEBUG [RS:1;a3dd13ba50c2:40427 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:26,077 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3dd13ba50c2,40427,1732712185325' 2024-11-27T12:56:26,077 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T12:56:26,078 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T12:56:26,078 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T12:56:26,078 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T12:56:26,078 DEBUG [RS:1;a3dd13ba50c2:40427 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:26,078 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3dd13ba50c2,40427,1732712185325' 2024-11-27T12:56:26,078 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T12:56:26,079 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T12:56:26,079 DEBUG [RS:1;a3dd13ba50c2:40427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T12:56:26,079 INFO [RS:1;a3dd13ba50c2:40427 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T12:56:26,079 INFO [RS:1;a3dd13ba50c2:40427 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T12:56:26,086 WARN [a3dd13ba50c2:43841 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T12:56:26,174 INFO [RS:0;a3dd13ba50c2:41341 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:56:26,175 INFO [RS:0;a3dd13ba50c2:41341 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a3dd13ba50c2%2C41341%2C1732712185256, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,41341,1732712185256, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs, maxLogs=55 2024-11-27T12:56:26,180 INFO [RS:1;a3dd13ba50c2:40427 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:56:26,181 INFO [RS:1;a3dd13ba50c2:40427 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a3dd13ba50c2%2C40427%2C1732712185325, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40427,1732712185325, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs, maxLogs=55 2024-11-27T12:56:26,197 INFO [RS:0;a3dd13ba50c2:41341 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,41341,1732712185256/a3dd13ba50c2%2C41341%2C1732712185256.1732712186176 2024-11-27T12:56:26,197 DEBUG [RS:0;a3dd13ba50c2:41341 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T12:56:26,201 INFO [RS:1;a3dd13ba50c2:40427 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40427,1732712185325/a3dd13ba50c2%2C40427%2C1732712185325.1732712186182 2024-11-27T12:56:26,203 DEBUG [RS:1;a3dd13ba50c2:40427 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T12:56:26,336 DEBUG [a3dd13ba50c2:43841 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-27T12:56:26,336 DEBUG [a3dd13ba50c2:43841 {}] balancer.BalancerClusterState(204): Hosts are {a3dd13ba50c2=0} racks are {/default-rack=0} 2024-11-27T12:56:26,337 DEBUG [a3dd13ba50c2:43841 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T12:56:26,337 DEBUG [a3dd13ba50c2:43841 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T12:56:26,337 DEBUG [a3dd13ba50c2:43841 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T12:56:26,337 DEBUG [a3dd13ba50c2:43841 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T12:56:26,337 INFO [a3dd13ba50c2:43841 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T12:56:26,337 INFO [a3dd13ba50c2:43841 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T12:56:26,337 DEBUG [a3dd13ba50c2:43841 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T12:56:26,338 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:26,339 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3dd13ba50c2,40427,1732712185325, state=OPENING 2024-11-27T12:56:26,358 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T12:56:26,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:26,433 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:26,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:26,434 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:26,434 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:26,434 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:56:26,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:26,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a3dd13ba50c2,40427,1732712185325}] 2024-11-27T12:56:26,588 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T12:56:26,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42583, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T12:56:26,594 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T12:56:26,594 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:56:26,594 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T12:56:26,595 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a3dd13ba50c2%2C40427%2C1732712185325.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40427,1732712185325, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs, maxLogs=55 2024-11-27T12:56:26,616 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40427,1732712185325/a3dd13ba50c2%2C40427%2C1732712185325.meta.1732712186596.meta 2024-11-27T12:56:26,618 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T12:56:26,618 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:56:26,619 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T12:56:26,619 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T12:56:26,619 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T12:56:26,619 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T12:56:26,619 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:26,619 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T12:56:26,620 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T12:56:26,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-27T12:56:26,623 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T12:56:26,623 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:26,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:26,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-27T12:56:26,624 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T12:56:26,624 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:26,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:26,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-27T12:56:26,625 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T12:56:26,625 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:26,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:26,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-27T12:56:26,626 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T12:56:26,626 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:26,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:26,626 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T12:56:26,627 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740 2024-11-27T12:56:26,628 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740 2024-11-27T12:56:26,628 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T12:56:26,628 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T12:56:26,628 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:56:26,629 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T12:56:26,630 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73576460, jitterRate=0.09637469053268433}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:56:26,630 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T12:56:26,631 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732712186622Writing region info on filesystem at 1732712186622Initializing all the Stores at 1732712186622Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712186622Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712186623 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712186623Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712186623Cleaning up temporary data from old regions at 1732712186628 (+5 ms)Running coprocessor post-open hooks at 1732712186630 (+2 ms)Region opened successfully at 1732712186631 (+1 ms) 2024-11-27T12:56:26,632 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732712186587 2024-11-27T12:56:26,634 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T12:56:26,635 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T12:56:26,635 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:26,635 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3dd13ba50c2,40427,1732712185325, state=OPEN 2024-11-27T12:56:26,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:56:26,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:56:26,650 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:56:26,650 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:26,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:26,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:26,651 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:26,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T12:56:26,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a3dd13ba50c2,40427,1732712185325 in 215 msec 2024-11-27T12:56:26,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T12:56:26,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 720 msec 2024-11-27T12:56:26,656 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:56:26,656 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T12:56:26,660 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:56:26,660 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3dd13ba50c2,40427,1732712185325, seqNum=-1] 2024-11-27T12:56:26,660 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:56:26,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45717, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:56:26,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 838 msec 2024-11-27T12:56:26,668 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732712186668, completionTime=-1 2024-11-27T12:56:26,669 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-27T12:56:26,669 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T12:56:26,672 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T12:56:26,672 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732712246672 2024-11-27T12:56:26,672 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732712306672 2024-11-27T12:56:26,672 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-27T12:56:26,673 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,43841,1732712185108-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,673 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,43841,1732712185108-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,673 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,43841,1732712185108-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,673 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a3dd13ba50c2:43841, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,673 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,674 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,677 DEBUG [master/a3dd13ba50c2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T12:56:26,679 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.228sec 2024-11-27T12:56:26,680 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T12:56:26,680 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T12:56:26,680 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T12:56:26,680 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T12:56:26,680 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T12:56:26,680 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,43841,1732712185108-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:56:26,680 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,43841,1732712185108-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T12:56:26,681 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T12:56:26,681 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T12:56:26,682 INFO [master/a3dd13ba50c2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,43841,1732712185108-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:26,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:56:26,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3dd13ba50c2,43841,-1 for getting cluster id 2024-11-27T12:56:26,703 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:56:26,706 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '49a81152-fef5-4818-bb52-858e1a5a1942' 2024-11-27T12:56:26,706 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:56:26,707 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "49a81152-fef5-4818-bb52-858e1a5a1942" 2024-11-27T12:56:26,707 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6325ca12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:56:26,707 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3dd13ba50c2,43841,-1] 2024-11-27T12:56:26,708 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:56:26,708 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:26,710 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:56:26,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:56:26,714 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:56:26,723 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3dd13ba50c2,40427,1732712185325, seqNum=-1] 2024-11-27T12:56:26,724 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:56:26,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:56:26,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a3dd13ba50c2,43841,1732712185108 2024-11-27T12:56:26,733 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T12:56:26,735 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is a3dd13ba50c2,43841,1732712185108 2024-11-27T12:56:26,735 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4e58b03d 2024-11-27T12:56:26,735 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T12:56:26,748 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T12:56:26,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43841 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732712186733', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:56:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43841 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732712186733 2024-11-27T12:56:26,752 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732712186733 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T12:56:26,753 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:26,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43841 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732712186733" procId is: 4 2024-11-27T12:56:26,754 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732712186733 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T12:56:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43841 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:56:26,795 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732712186733-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 26ce3555746db7891e983f64e97d670a, NAME => 'testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732712186733', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6 2024-11-27T12:56:26,823 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732712186733-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:26,823 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732712186733-pool-0 {}] regionserver.HRegion(1722): Closing 26ce3555746db7891e983f64e97d670a, disabling compactions & flushes 2024-11-27T12:56:26,823 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732712186733-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:26,823 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732712186733-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:26,823 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732712186733-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. after waiting 0 ms 2024-11-27T12:56:26,824 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732712186733-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:26,824 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732712186733-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:26,824 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732712186733-pool-0 {}] regionserver.HRegion(1676): Region close journal for 26ce3555746db7891e983f64e97d670a: Waiting for close lock at 1732712186823Disabling compacts and flushes for region at 1732712186823Disabling writes for close at 1732712186823Writing region close event to WAL at 1732712186824 (+1 ms)Closed at 1732712186824 2024-11-27T12:56:26,825 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732712186733 execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T12:56:26,825 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732712186825"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732712186825"}]},"ts":"1732712186825"} 2024-11-27T12:56:26,831 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T12:56:26,833 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732712186733 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T12:56:26,833 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732712186733","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732712186833"}]},"ts":"1732712186833"} 2024-11-27T12:56:26,836 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732712186733, state=ENABLING in hbase:meta 2024-11-27T12:56:26,837 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a3dd13ba50c2=0} racks are {/default-rack=0} 2024-11-27T12:56:26,837 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T12:56:26,838 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T12:56:26,838 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T12:56:26,838 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T12:56:26,838 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T12:56:26,838 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T12:56:26,838 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T12:56:26,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732712186733, region=26ce3555746db7891e983f64e97d670a, ASSIGN}] 2024-11-27T12:56:26,840 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732712186733, region=26ce3555746db7891e983f64e97d670a, ASSIGN 2024-11-27T12:56:26,841 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732712186733, region=26ce3555746db7891e983f64e97d670a, ASSIGN; state=OFFLINE, location=a3dd13ba50c2,40427,1732712185325; forceNewPlan=false, retain=false 2024-11-27T12:56:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43841 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:56:26,992 INFO [a3dd13ba50c2:43841 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T12:56:26,993 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=26ce3555746db7891e983f64e97d670a, regionState=OPENING, regionLocation=a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:26,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732712186733, region=26ce3555746db7891e983f64e97d670a, ASSIGN because future has completed 2024-11-27T12:56:26,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26ce3555746db7891e983f64e97d670a, server=a3dd13ba50c2,40427,1732712185325}] 2024-11-27T12:56:27,046 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T12:56:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43841 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:56:27,151 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:27,151 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 26ce3555746db7891e983f64e97d670a, NAME => 'testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:56:27,152 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732712186733 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:27,152 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:27,152 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:27,152 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:27,153 INFO [StoreOpener-26ce3555746db7891e983f64e97d670a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:27,157 INFO [StoreOpener-26ce3555746db7891e983f64e97d670a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26ce3555746db7891e983f64e97d670a columnFamilyName family 2024-11-27T12:56:27,158 DEBUG [StoreOpener-26ce3555746db7891e983f64e97d670a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:27,158 INFO [StoreOpener-26ce3555746db7891e983f64e97d670a-1 {}] regionserver.HStore(327): Store=26ce3555746db7891e983f64e97d670a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:56:27,158 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:27,159 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:27,159 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:27,159 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:27,159 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:27,160 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:27,167 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:56:27,167 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 26ce3555746db7891e983f64e97d670a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73989872, jitterRate=0.10253500938415527}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:56:27,168 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:27,169 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 26ce3555746db7891e983f64e97d670a: Running coprocessor pre-open hook at 1732712187152Writing region info on filesystem at 1732712187152Initializing all the Stores at 1732712187153 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712187153Cleaning up temporary data from old regions at 1732712187159 (+6 ms)Running coprocessor post-open hooks at 1732712187168 (+9 ms)Region opened successfully at 1732712187168 2024-11-27T12:56:27,169 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a., pid=6, masterSystemTime=1732712187149 2024-11-27T12:56:27,174 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=26ce3555746db7891e983f64e97d670a, regionState=OPEN, openSeqNum=2, regionLocation=a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:27,175 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:27,175 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:27,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26ce3555746db7891e983f64e97d670a, server=a3dd13ba50c2,40427,1732712185325 because future has completed 2024-11-27T12:56:27,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T12:56:27,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 26ce3555746db7891e983f64e97d670a, server=a3dd13ba50c2,40427,1732712185325 in 189 msec 2024-11-27T12:56:27,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T12:56:27,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732712186733, region=26ce3555746db7891e983f64e97d670a, ASSIGN in 352 msec 2024-11-27T12:56:27,194 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732712186733 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T12:56:27,194 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732712186733","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732712187194"}]},"ts":"1732712187194"} 2024-11-27T12:56:27,197 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732712186733, state=ENABLED in hbase:meta 2024-11-27T12:56:27,198 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732712186733 execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T12:56:27,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732712186733 in 449 msec 2024-11-27T12:56:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43841 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:56:27,387 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732712186733 completed 2024-11-27T12:56:27,387 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732712186733 get assigned. Timeout = 60000ms 2024-11-27T12:56:27,388 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T12:56:27,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732712186733 assigned to meta. Checking AM states. 2024-11-27T12:56:27,392 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T12:56:27,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732712186733 assigned. 2024-11-27T12:56:27,397 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732712186733', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a., hostname=a3dd13ba50c2,40427,1732712185325, seqNum=2] 2024-11-27T12:56:27,402 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 26ce3555746db7891e983f64e97d670a 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-27T12:56:27,412 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 17a2efba334047398c11c41caaa3db26 2024-11-27T12:56:27,425 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/.tmp/family/17a2efba334047398c11c41caaa3db26 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-27T12:56:27,426 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/.tmp/family/17a2efba334047398c11c41caaa3db26 2024-11-27T12:56:27,430 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/.tmp/family/17a2efba334047398c11c41caaa3db26 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26 2024-11-27T12:56:27,431 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26, delay=940 ms 2024-11-27T12:56:27,432 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26, entries=2, sequenceid=6, filesize=4.9 K 2024-11-27T12:56:27,433 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 26ce3555746db7891e983f64e97d670a in 31ms, sequenceid=6, compaction requested=false 2024-11-27T12:56:27,433 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732712186733' 2024-11-27T12:56:27,434 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 26ce3555746db7891e983f64e97d670a: 2024-11-27T12:56:28,233 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732712186733 2024-11-27T12:56:28,234 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732712186733 Metrics about Tables on a single HBase RegionServer 2024-11-27T12:56:28,237 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T12:56:28,237 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T12:56:28,238 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T12:56:28,238 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T12:56:28,239 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T12:56:28,239 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T12:56:28,372 DEBUG [hfile-prefetch-1732712184392 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 17a2efba334047398c11c41caaa3db26_0. 2024-11-27T12:56:28,373 DEBUG [hfile-prefetch-1732712184392 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 17a2efba334047398c11c41caaa3db26_117. 2024-11-27T12:56:28,373 DEBUG [hfile-prefetch-1732712184392 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26, with total blocks 2, and data blocks 1 2024-11-27T12:56:28,373 DEBUG [hfile-prefetch-1732712184392 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-27T12:56:28,373 DEBUG [hfile-prefetch-1732712184392 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-27T12:56:28,374 DEBUG [hfile-prefetch-1732712184392 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26. Will sleep 100ms and try the verification again. 2024-11-27T12:56:28,435 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;a3dd13ba50c2:40427,5,FailOnTimeoutGroup] 2024-11-27T12:56:28,435 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a3dd13ba50c2,40427,1732712185325' ***** 2024-11-27T12:56:28,435 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-27T12:56:28,435 INFO [RS:1;a3dd13ba50c2:40427 {}] bucket.BucketCache(1738): Disabling cache 2024-11-27T12:56:28,436 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T12:56:28,440 INFO [regionserver/a3dd13ba50c2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:56:28,464 INFO [RS:1;a3dd13ba50c2:40427 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.persistence 2024-11-27T12:56:28,464 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-27T12:56:28,464 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-27T12:56:28,464 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-27T12:56:28,464 INFO [RS:1;a3dd13ba50c2:40427 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-27T12:56:28,465 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-27T12:56:28,474 DEBUG [hfile-prefetch-1732712184392 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26, with total blocks 2, and data blocks 1 2024-11-27T12:56:28,474 DEBUG [hfile-prefetch-1732712184392 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-27T12:56:28,474 DEBUG [hfile-prefetch-1732712184392 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-27T12:56:28,474 INFO [hfile-prefetch-1732712184392 {}] bucket.BucketCache(2331): We found only 0 data blocks cached from a total of 1 for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch. 2024-11-27T12:56:28,474 DEBUG [hfile-prefetch-1732712184392 {}] hfile.PrefetchExecutor(121): Prefetch completed for 17a2efba334047398c11c41caaa3db26 2024-11-27T12:56:28,475 DEBUG [RS:1;a3dd13ba50c2:40427 {}] bucket.BucketCache(1487): Thread RS:1;a3dd13ba50c2:40427 started persisting bucket cache to file 2024-11-27T12:56:28,475 DEBUG [RS:1;a3dd13ba50c2:40427 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T12:56:28,475 DEBUG [RS:1;a3dd13ba50c2:40427 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T12:56:28,478 DEBUG [RS:1;a3dd13ba50c2:40427 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T12:56:28,478 DEBUG [RS:1;a3dd13ba50c2:40427 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732712188475 2024-11-27T12:56:28,478 DEBUG [RS:1;a3dd13ba50c2:40427 {}] bucket.BucketCache(1509): Thread RS:1;a3dd13ba50c2:40427 finished persisting bucket cache to file, renaming 2024-11-27T12:56:28,478 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T12:56:28,478 INFO [RS:1;a3dd13ba50c2:40427 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T12:56:28,478 INFO [RS:1;a3dd13ba50c2:40427 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T12:56:28,478 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(3091): Received CLOSE for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:28,479 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(959): stopping server a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:28,479 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:56:28,479 INFO [RS:1;a3dd13ba50c2:40427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a3dd13ba50c2:40427. 2024-11-27T12:56:28,479 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-27T12:56:28,479 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 26ce3555746db7891e983f64e97d670a, disabling compactions & flushes 2024-11-27T12:56:28,479 DEBUG [RS:1;a3dd13ba50c2:40427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:56:28,479 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:28,479 DEBUG [RS:1;a3dd13ba50c2:40427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:28,479 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:28,479 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. after waiting 0 ms 2024-11-27T12:56:28,479 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T12:56:28,479 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:28,479 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T12:56:28,479 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T12:56:28,479 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T12:56:28,479 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-27T12:56:28,480 DEBUG [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(1325): Online Regions={26ce3555746db7891e983f64e97d670a=testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a., 1588230740=hbase:meta,,1.1588230740} 2024-11-27T12:56:28,480 DEBUG [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:28,480 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-27T12:56:28,480 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T12:56:28,480 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T12:56:28,480 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T12:56:28,480 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T12:56:28,480 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T12:56:28,480 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-27T12:56:28,481 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-27T12:56:28,481 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-27T12:56:28,481 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T12:56:28,481 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T12:56:28,492 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T12:56:28,492 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-27T12:56:28,492 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732712188481 2024-11-27T12:56:28,492 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-27T12:56:28,493 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:28,493 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 26ce3555746db7891e983f64e97d670a: Waiting for close lock at 1732712188479Running coprocessor pre-close hooks at 1732712188479Disabling compacts and flushes for region at 1732712188479Disabling writes for close at 1732712188479Writing region close event to WAL at 1732712188480 (+1 ms)Running coprocessor post-close hooks at 1732712188493 (+13 ms)Closed at 1732712188493 2024-11-27T12:56:28,493 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:28,499 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8e4cfb98f131438b95443aefa379d930 2024-11-27T12:56:28,517 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/info/8e4cfb98f131438b95443aefa379d930 is 209, key is testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a./info:regioninfo/1732712187174/Put/seqid=0 2024-11-27T12:56:28,518 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/info/8e4cfb98f131438b95443aefa379d930 2024-11-27T12:56:28,530 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 769b1265b78c48e2a37972e923fe607a 2024-11-27T12:56:28,547 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/ns/769b1265b78c48e2a37972e923fe607a is 43, key is default/ns:d/1732712186662/Put/seqid=0 2024-11-27T12:56:28,548 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/ns/769b1265b78c48e2a37972e923fe607a 2024-11-27T12:56:28,564 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 69774ee77b4a408b98802d994f51c6ab 2024-11-27T12:56:28,583 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/table/69774ee77b4a408b98802d994f51c6ab is 80, key is testBlockEvictionOnGracefulStop1732712186733/table:state/1732712187194/Put/seqid=0 2024-11-27T12:56:28,584 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/table/69774ee77b4a408b98802d994f51c6ab 2024-11-27T12:56:28,586 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/info/8e4cfb98f131438b95443aefa379d930 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/8e4cfb98f131438b95443aefa379d930 2024-11-27T12:56:28,588 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/8e4cfb98f131438b95443aefa379d930, delay=966 ms 2024-11-27T12:56:28,589 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/8e4cfb98f131438b95443aefa379d930, entries=10, sequenceid=11, filesize=7.0 K 2024-11-27T12:56:28,590 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/ns/769b1265b78c48e2a37972e923fe607a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/ns/769b1265b78c48e2a37972e923fe607a 2024-11-27T12:56:28,591 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/ns/769b1265b78c48e2a37972e923fe607a, delay=996 ms 2024-11-27T12:56:28,592 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/ns/769b1265b78c48e2a37972e923fe607a, entries=2, sequenceid=11, filesize=5.0 K 2024-11-27T12:56:28,592 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/table/69774ee77b4a408b98802d994f51c6ab as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/table/69774ee77b4a408b98802d994f51c6ab 2024-11-27T12:56:28,593 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/table/69774ee77b4a408b98802d994f51c6ab, delay=964 ms 2024-11-27T12:56:28,595 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/table/69774ee77b4a408b98802d994f51c6ab, entries=2, sequenceid=11, filesize=5.3 K 2024-11-27T12:56:28,598 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 117ms, sequenceid=11, compaction requested=false 2024-11-27T12:56:28,598 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/8e4cfb98f131438b95443aefa379d930 2024-11-27T12:56:28,602 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/ns/769b1265b78c48e2a37972e923fe607a 2024-11-27T12:56:28,607 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/table/69774ee77b4a408b98802d994f51c6ab 2024-11-27T12:56:28,615 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-27T12:56:28,615 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T12:56:28,615 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T12:56:28,615 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732712188480Running coprocessor pre-close hooks at 1732712188480Disabling compacts and flushes for region at 1732712188480Disabling writes for close at 1732712188480Obtaining lock to block concurrent updates at 1732712188480Preparing flush snapshotting stores in 1588230740 at 1732712188480Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732712188481 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732712188481Flushing 1588230740/info: creating writer at 1732712188481Flushing 1588230740/info: appending metadata at 1732712188516 (+35 ms)Flushing 1588230740/info: closing flushed file at 1732712188516Flushing 1588230740/ns: creating writer at 1732712188520 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732712188547 (+27 ms)Flushing 1588230740/ns: closing flushed file at 1732712188547Flushing 1588230740/table: creating writer at 1732712188551 (+4 ms)Flushing 1588230740/table: appending metadata at 1732712188582 (+31 ms)Flushing 1588230740/table: closing flushed file at 1732712188582Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6daa0b2d: reopening flushed file at 1732712188586 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55cda80: reopening flushed file at 1732712188589 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cf73f1b: reopening flushed file at 1732712188592 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 117ms, sequenceid=11, compaction requested=false at 1732712188598 (+6 ms)Writing region close event to WAL at 1732712188609 (+11 ms)Running coprocessor post-close hooks at 1732712188615 (+6 ms)Closed at 1732712188615 2024-11-27T12:56:28,616 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T12:56:28,680 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(976): stopping server a3dd13ba50c2,40427,1732712185325; all regions closed. 2024-11-27T12:56:28,682 DEBUG [RS:1;a3dd13ba50c2:40427 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs 2024-11-27T12:56:28,682 INFO [RS:1;a3dd13ba50c2:40427 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3dd13ba50c2%2C40427%2C1732712185325.meta:.meta(num 1732712186596) 2024-11-27T12:56:28,684 DEBUG [RS:1;a3dd13ba50c2:40427 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs 2024-11-27T12:56:28,684 INFO [RS:1;a3dd13ba50c2:40427 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3dd13ba50c2%2C40427%2C1732712185325:(num 1732712186182) 2024-11-27T12:56:28,684 DEBUG [RS:1;a3dd13ba50c2:40427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:28,684 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:56:28,684 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:56:28,685 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.ChoreService(370): Chore service for: regionserver/a3dd13ba50c2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T12:56:28,685 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:56:28,685 INFO [regionserver/a3dd13ba50c2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:56:28,685 INFO [RS:1;a3dd13ba50c2:40427 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40427 2024-11-27T12:56:28,717 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a3dd13ba50c2,40427,1732712185325 2024-11-27T12:56:28,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:56:28,717 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:56:28,717 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a3dd13ba50c2,40427,1732712185325] 2024-11-27T12:56:28,733 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a3dd13ba50c2,40427,1732712185325 already deleted, retry=false 2024-11-27T12:56:28,733 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a3dd13ba50c2,40427,1732712185325 on a3dd13ba50c2,43841,1732712185108 2024-11-27T12:56:28,737 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a3dd13ba50c2,40427,1732712185325, splitWal=true, meta=true 2024-11-27T12:56:28,739 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for a3dd13ba50c2,40427,1732712185325 (carryingMeta=true) a3dd13ba50c2,40427,1732712185325/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6a0c4f58[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-27T12:56:28,740 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a3dd13ba50c2,40427,1732712185325, splitWal=true, meta=true 2024-11-27T12:56:28,741 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure a3dd13ba50c2,40427,1732712185325, splitWal=true, meta=true, isMeta: true 2024-11-27T12:56:28,742 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40427,1732712185325-splitting 2024-11-27T12:56:28,742 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40427,1732712185325-splitting dir is empty, no logs to split. 2024-11-27T12:56:28,742 INFO [PEWorker-2 {}] master.SplitWALManager(105): a3dd13ba50c2,40427,1732712185325 WAL count=0, meta=true 2024-11-27T12:56:28,744 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40427,1732712185325-splitting dir is empty, no logs to split. 2024-11-27T12:56:28,744 INFO [PEWorker-2 {}] master.SplitWALManager(105): a3dd13ba50c2,40427,1732712185325 WAL count=0, meta=true 2024-11-27T12:56:28,744 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if a3dd13ba50c2,40427,1732712185325 WAL splitting is done? wals=0, meta=true 2024-11-27T12:56:28,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T12:56:28,747 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:56:28,748 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T12:56:28,825 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:28,825 INFO [RS:1;a3dd13ba50c2:40427 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:56:28,825 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x1017da74fca0002, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:28,825 INFO [RS:1;a3dd13ba50c2:40427 {}] regionserver.HRegionServer(1031): Exiting; stopping=a3dd13ba50c2,40427,1732712185325; zookeeper connection closed. 2024-11-27T12:56:28,825 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67db4b44 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67db4b44 2024-11-27T12:56:28,899 DEBUG [a3dd13ba50c2:43841 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T12:56:28,899 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:28,900 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3dd13ba50c2,41341,1732712185256, state=OPENING 2024-11-27T12:56:28,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:56:28,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:56:28,908 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:56:28,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:28,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:28,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a3dd13ba50c2,41341,1732712185256}] 2024-11-27T12:56:28,951 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a3dd13ba50c2:0 server-side Connection retries=45 2024-11-27T12:56:28,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:28,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:28,951 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:56:28,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:56:28,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:56:28,952 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T12:56:28,952 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:56:28,952 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40641 2024-11-27T12:56:28,954 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40641 connecting to ZooKeeper ensemble=127.0.0.1:52261 2024-11-27T12:56:28,954 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:28,954 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:28,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406410x0, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:56:28,980 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:406410x0, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-27T12:56:28,980 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40641-0x1017da74fca0003 connected 2024-11-27T12:56:28,981 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-27T12:56:28,981 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T12:56:28,983 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.cache 2024-11-27T12:56:28,984 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-27T12:56:28,984 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-27T12:56:28,984 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-27T12:56:28,984 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-27T12:56:28,985 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-27T12:56:28,986 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-27T12:56:28,986 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-27T12:56:28,988 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T12:56:28,989 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40641-0x1017da74fca0003, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T12:56:28,992 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40641-0x1017da74fca0003, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:56:29,012 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-27T12:56:29,013 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-27T12:56:29,013 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-27T12:56:29,013 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.persistence is complete. 2024-11-27T12:56:29,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40641 2024-11-27T12:56:29,017 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40641 2024-11-27T12:56:29,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40641 2024-11-27T12:56:29,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40641 2024-11-27T12:56:29,031 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40641 2024-11-27T12:56:29,036 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer(746): ClusterId : 49a81152-fef5-4818-bb52-858e1a5a1942 2024-11-27T12:56:29,036 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T12:56:29,061 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T12:56:29,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35179, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T12:56:29,068 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T12:56:29,068 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:56:29,069 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T12:56:29,070 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a3dd13ba50c2%2C41341%2C1732712185256.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,41341,1732712185256, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs, maxLogs=55 2024-11-27T12:56:29,071 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T12:56:29,071 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T12:56:29,081 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T12:56:29,082 DEBUG [RS:2;a3dd13ba50c2:40641 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3812dc7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3dd13ba50c2/172.17.0.2:0 2024-11-27T12:56:29,095 DEBUG [RS:2;a3dd13ba50c2:40641 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;a3dd13ba50c2:40641 2024-11-27T12:56:29,096 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T12:56:29,096 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T12:56:29,096 DEBUG [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T12:56:29,096 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer(2659): reportForDuty to master=a3dd13ba50c2,43841,1732712185108 with port=40641, startcode=1732712188951 2024-11-27T12:56:29,097 DEBUG [RS:2;a3dd13ba50c2:40641 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T12:56:29,099 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47101, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T12:56:29,099 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43841 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a3dd13ba50c2,40641,1732712188951 2024-11-27T12:56:29,099 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43841 {}] master.ServerManager(517): Registering regionserver=a3dd13ba50c2,40641,1732712188951 2024-11-27T12:56:29,102 DEBUG [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6 2024-11-27T12:56:29,102 DEBUG [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-27T12:56:29,102 DEBUG [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T12:56:29,103 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,41341,1732712185256/a3dd13ba50c2%2C41341%2C1732712185256.meta.1732712189072.meta 2024-11-27T12:56:29,103 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T12:56:29,103 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:56:29,104 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T12:56:29,104 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T12:56:29,104 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T12:56:29,104 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T12:56:29,104 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:29,104 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T12:56:29,104 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T12:56:29,107 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-27T12:56:29,107 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T12:56:29,107 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:29,110 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/8e4cfb98f131438b95443aefa379d930, delay=924 ms 2024-11-27T12:56:29,111 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/8e4cfb98f131438b95443aefa379d930 2024-11-27T12:56:29,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:29,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-27T12:56:29,112 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T12:56:29,112 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:29,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:56:29,114 DEBUG [RS:2;a3dd13ba50c2:40641 {}] zookeeper.ZKUtil(111): regionserver:40641-0x1017da74fca0003, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a3dd13ba50c2,40641,1732712188951 2024-11-27T12:56:29,114 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/ns/769b1265b78c48e2a37972e923fe607a, delay=943 ms 2024-11-27T12:56:29,114 WARN [RS:2;a3dd13ba50c2:40641 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:56:29,114 INFO [RS:2;a3dd13ba50c2:40641 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:56:29,114 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a3dd13ba50c2,40641,1732712188951] 2024-11-27T12:56:29,114 DEBUG [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40641,1732712188951 2024-11-27T12:56:29,115 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/ns/769b1265b78c48e2a37972e923fe607a 2024-11-27T12:56:29,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:29,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-27T12:56:29,117 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T12:56:29,117 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:29,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:29,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-27T12:56:29,118 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T12:56:29,118 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:29,120 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/table/69774ee77b4a408b98802d994f51c6ab, delay=917 ms 2024-11-27T12:56:29,120 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T12:56:29,121 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/table/69774ee77b4a408b98802d994f51c6ab 2024-11-27T12:56:29,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:56:29,121 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T12:56:29,122 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740 2024-11-27T12:56:29,122 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740 2024-11-27T12:56:29,123 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T12:56:29,123 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T12:56:29,123 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T12:56:29,123 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:56:29,124 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T12:56:29,125 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74470223, jitterRate=0.10969279706478119}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:56:29,125 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T12:56:29,126 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732712189104Writing region info on filesystem at 1732712189104Initializing all the Stores at 1732712189105 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712189105Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712189106 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712189106Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732712189106Cleaning up temporary data from old regions at 1732712189123 (+17 ms)Running coprocessor post-open hooks at 1732712189125 (+2 ms)Region opened successfully at 1732712189126 (+1 ms) 2024-11-27T12:56:29,127 INFO [RS:2;a3dd13ba50c2:40641 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T12:56:29,127 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:29,128 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732712189061 2024-11-27T12:56:29,130 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T12:56:29,131 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T12:56:29,131 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:29,131 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:29,131 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:29,132 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:29,132 DEBUG [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T12:56:29,132 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:29,132 INFO [RS_OPEN_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T12:56:29,132 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:29,132 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a3dd13ba50c2:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:56:29,132 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:29,132 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:29,132 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:29,132 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:29,132 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:29,132 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a3dd13ba50c2:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:56:29,132 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a3dd13ba50c2:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:56:29,133 DEBUG [RS:2;a3dd13ba50c2:40641 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a3dd13ba50c2:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:56:29,134 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:29,134 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3dd13ba50c2,41341,1732712185256, state=OPEN 2024-11-27T12:56:29,138 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:29,139 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:29,139 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:29,139 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:29,139 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:29,139 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,40641,1732712188951-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:56:29,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:56:29,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40641-0x1017da74fca0003, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:56:29,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:56:29,141 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:29,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:29,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:29,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:56:29,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-27T12:56:29,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a3dd13ba50c2,41341,1732712185256 in 234 msec 2024-11-27T12:56:29,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T12:56:29,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 398 msec 2024-11-27T12:56:29,147 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(207): a3dd13ba50c2,40427,1732712185325 had 2 regions 2024-11-27T12:56:29,148 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure a3dd13ba50c2,40427,1732712185325, splitWal=true, meta=true, isMeta: false 2024-11-27T12:56:29,149 INFO [PEWorker-4 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40427,1732712185325-splitting dir is empty, no logs to split. 2024-11-27T12:56:29,149 INFO [PEWorker-4 {}] master.SplitWALManager(105): a3dd13ba50c2,40427,1732712185325 WAL count=0, meta=false 2024-11-27T12:56:29,150 INFO [PEWorker-4 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40427,1732712185325-splitting dir is empty, no logs to split. 2024-11-27T12:56:29,150 INFO [PEWorker-4 {}] master.SplitWALManager(105): a3dd13ba50c2,40427,1732712185325 WAL count=0, meta=false 2024-11-27T12:56:29,150 DEBUG [PEWorker-4 {}] procedure.ServerCrashProcedure(329): Check if a3dd13ba50c2,40427,1732712185325 WAL splitting is done? wals=0, meta=false 2024-11-27T12:56:29,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732712186733, region=26ce3555746db7891e983f64e97d670a, ASSIGN}] 2024-11-27T12:56:29,153 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732712186733, region=26ce3555746db7891e983f64e97d670a, ASSIGN 2024-11-27T12:56:29,154 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732712186733, region=26ce3555746db7891e983f64e97d670a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T12:56:29,157 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T12:56:29,158 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3dd13ba50c2,40641,1732712188951-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:29,158 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:29,158 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.Replication(171): a3dd13ba50c2,40641,1732712188951 started 2024-11-27T12:56:29,176 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:56:29,176 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer(1482): Serving as a3dd13ba50c2,40641,1732712188951, RpcServer on a3dd13ba50c2/172.17.0.2:40641, sessionid=0x1017da74fca0003 2024-11-27T12:56:29,176 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T12:56:29,176 DEBUG [RS:2;a3dd13ba50c2:40641 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a3dd13ba50c2,40641,1732712188951 2024-11-27T12:56:29,176 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3dd13ba50c2,40641,1732712188951' 2024-11-27T12:56:29,176 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T12:56:29,177 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T12:56:29,177 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T12:56:29,177 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T12:56:29,177 DEBUG [RS:2;a3dd13ba50c2:40641 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a3dd13ba50c2,40641,1732712188951 2024-11-27T12:56:29,177 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3dd13ba50c2,40641,1732712188951' 2024-11-27T12:56:29,177 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T12:56:29,177 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T12:56:29,178 DEBUG [RS:2;a3dd13ba50c2:40641 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T12:56:29,178 INFO [RS:2;a3dd13ba50c2:40641 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T12:56:29,178 INFO [RS:2;a3dd13ba50c2:40641 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T12:56:29,279 INFO [RS:2;a3dd13ba50c2:40641 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:56:29,280 INFO [RS:2;a3dd13ba50c2:40641 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a3dd13ba50c2%2C40641%2C1732712188951, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40641,1732712188951, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs, maxLogs=55 2024-11-27T12:56:29,303 INFO [RS:2;a3dd13ba50c2:40641 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/WALs/a3dd13ba50c2,40641,1732712188951/a3dd13ba50c2%2C40641%2C1732712188951.1732712189281 2024-11-27T12:56:29,304 DEBUG [RS:2;a3dd13ba50c2:40641 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T12:56:29,305 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=26ce3555746db7891e983f64e97d670a, regionState=OPENING, regionLocation=a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:29,306 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a3dd13ba50c2:40427 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a3dd13ba50c2/172.17.0.2:40427 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:56:29,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a3dd13ba50c2,40427,1732712185325, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a3dd13ba50c2,40427,1732712185325, seqNum=-1, error=java.net.ConnectException: Call to address=a3dd13ba50c2:40427 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a3dd13ba50c2/172.17.0.2:40427 2024-11-27T12:56:29,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a3dd13ba50c2,40427,1732712185325, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-27T12:56:29,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a3dd13ba50c2,40427,1732712185325, seqNum=-1 from cache 2024-11-27T12:56:29,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address a3dd13ba50c2:40427 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a3dd13ba50c2/172.17.0.2:40427 2024-11-27T12:56:29,415 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:56:29,415 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3dd13ba50c2,41341,1732712185256, seqNum=-1] 2024-11-27T12:56:29,415 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:56:29,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33311, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:56:29,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732712186733, region=26ce3555746db7891e983f64e97d670a, ASSIGN because future has completed 2024-11-27T12:56:29,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26ce3555746db7891e983f64e97d670a, server=a3dd13ba50c2,41341,1732712185256}] 2024-11-27T12:56:29,576 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:29,577 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 26ce3555746db7891e983f64e97d670a, NAME => 'testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:56:29,577 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732712186733 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,577 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:56:29,577 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,577 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,578 INFO [StoreOpener-26ce3555746db7891e983f64e97d670a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,579 INFO [StoreOpener-26ce3555746db7891e983f64e97d670a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26ce3555746db7891e983f64e97d670a columnFamilyName family 2024-11-27T12:56:29,579 DEBUG [StoreOpener-26ce3555746db7891e983f64e97d670a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:56:29,581 DEBUG [StoreFileOpener-26ce3555746db7891e983f64e97d670a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26, delay=951 ms 2024-11-27T12:56:29,582 DEBUG [StoreOpener-26ce3555746db7891e983f64e97d670a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26 2024-11-27T12:56:29,582 INFO [StoreOpener-26ce3555746db7891e983f64e97d670a-1 {}] regionserver.HStore(327): Store=26ce3555746db7891e983f64e97d670a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:56:29,582 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,583 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,584 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,584 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,584 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,586 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,586 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 26ce3555746db7891e983f64e97d670a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75101232, jitterRate=0.1190955638885498}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:56:29,587 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,588 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 26ce3555746db7891e983f64e97d670a: Running coprocessor pre-open hook at 1732712189578Writing region info on filesystem at 1732712189578Initializing all the Stores at 1732712189578Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732712189578Cleaning up temporary data from old regions at 1732712189584 (+6 ms)Running coprocessor post-open hooks at 1732712189587 (+3 ms)Region opened successfully at 1732712189587 2024-11-27T12:56:29,588 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a., pid=11, masterSystemTime=1732712189574 2024-11-27T12:56:29,591 DEBUG [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:29,591 INFO [RS_OPEN_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:29,591 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=26ce3555746db7891e983f64e97d670a, regionState=OPEN, openSeqNum=10, regionLocation=a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:29,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26ce3555746db7891e983f64e97d670a, server=a3dd13ba50c2,41341,1732712185256 because future has completed 2024-11-27T12:56:29,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-27T12:56:29,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 26ce3555746db7891e983f64e97d670a, server=a3dd13ba50c2,41341,1732712185256 in 174 msec 2024-11-27T12:56:29,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-27T12:56:29,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732712186733, region=26ce3555746db7891e983f64e97d670a, ASSIGN in 448 msec 2024-11-27T12:56:29,601 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server a3dd13ba50c2,40427,1732712185325 after splitting done 2024-11-27T12:56:29,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure a3dd13ba50c2,40427,1732712185325, splitWal=true, meta=true in 867 msec 2024-11-27T12:56:29,676 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T12:56:29,676 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:56:29,676 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:56:29,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:29,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:29,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:56:29,677 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T12:56:29,677 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=704732517, stopped=false 2024-11-27T12:56:29,677 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a3dd13ba50c2,43841,1732712185108 2024-11-27T12:56:29,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:29,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40641-0x1017da74fca0003, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:29,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T12:56:29,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40641-0x1017da74fca0003, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:29,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:29,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:29,725 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T12:56:29,725 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:56:29,725 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:56:29,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:29,726 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a3dd13ba50c2,41341,1732712185256' ***** 2024-11-27T12:56:29,726 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T12:56:29,726 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a3dd13ba50c2,40641,1732712188951' ***** 2024-11-27T12:56:29,726 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T12:56:29,726 INFO [RS:0;a3dd13ba50c2:41341 {}] bucket.BucketCache(1738): Disabling cache 2024-11-27T12:56:29,726 INFO [RS:2;a3dd13ba50c2:40641 {}] bucket.BucketCache(1738): Disabling cache 2024-11-27T12:56:29,726 INFO [RS:0;a3dd13ba50c2:41341 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.persistence 2024-11-27T12:56:29,726 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-27T12:56:29,726 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-27T12:56:29,726 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-27T12:56:29,726 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-27T12:56:29,726 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-27T12:56:29,726 INFO [RS:0;a3dd13ba50c2:41341 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-27T12:56:29,726 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-27T12:56:29,726 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-27T12:56:29,728 DEBUG [RS:0;a3dd13ba50c2:41341 {}] bucket.BucketCache(1487): Thread RS:0;a3dd13ba50c2:41341 started persisting bucket cache to file 2024-11-27T12:56:29,729 DEBUG [RS:0;a3dd13ba50c2:41341 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T12:56:29,729 DEBUG [RS:0;a3dd13ba50c2:41341 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T12:56:29,729 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40641-0x1017da74fca0003, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:29,729 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:29,729 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T12:56:29,729 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:56:29,729 INFO [RS:2;a3dd13ba50c2:40641 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b972c348-1344-d69f-e6c7-e6b910252efd/bucket.persistence 2024-11-27T12:56:29,729 INFO [RS:2;a3dd13ba50c2:40641 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-27T12:56:29,729 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-27T12:56:29,732 DEBUG [RS:0;a3dd13ba50c2:41341 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T12:56:29,733 DEBUG [RS:0;a3dd13ba50c2:41341 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732712189729 2024-11-27T12:56:29,733 DEBUG [RS:0;a3dd13ba50c2:41341 {}] bucket.BucketCache(1509): Thread RS:0;a3dd13ba50c2:41341 finished persisting bucket cache to file, renaming 2024-11-27T12:56:29,733 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T12:56:29,733 INFO [RS:0;a3dd13ba50c2:41341 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T12:56:29,733 INFO [RS:0;a3dd13ba50c2:41341 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T12:56:29,733 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(3091): Received CLOSE for 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,733 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(959): stopping server a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:29,734 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:56:29,734 INFO [RS:0;a3dd13ba50c2:41341 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a3dd13ba50c2:41341. 2024-11-27T12:56:29,734 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-27T12:56:29,734 DEBUG [RS:0;a3dd13ba50c2:41341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:56:29,734 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 26ce3555746db7891e983f64e97d670a, disabling compactions & flushes 2024-11-27T12:56:29,734 DEBUG [RS:0;a3dd13ba50c2:41341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:29,734 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:29,734 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:29,734 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T12:56:29,734 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. after waiting 0 ms 2024-11-27T12:56:29,734 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T12:56:29,734 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:29,734 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T12:56:29,734 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T12:56:29,735 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-27T12:56:29,735 DEBUG [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(1325): Online Regions={26ce3555746db7891e983f64e97d670a=testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a., 1588230740=hbase:meta,,1.1588230740} 2024-11-27T12:56:29,735 DEBUG [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 26ce3555746db7891e983f64e97d670a 2024-11-27T12:56:29,735 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-27T12:56:29,735 INFO [regionserver/a3dd13ba50c2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:56:29,735 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T12:56:29,735 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T12:56:29,735 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T12:56:29,735 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T12:56:29,735 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T12:56:29,735 DEBUG [StoreFileCloser-26ce3555746db7891e983f64e97d670a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/family/17a2efba334047398c11c41caaa3db26 2024-11-27T12:56:29,735 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-27T12:56:29,736 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T12:56:29,739 DEBUG [RS:2;a3dd13ba50c2:40641 {}] bucket.BucketCache(1487): Thread RS:2;a3dd13ba50c2:40641 started persisting bucket cache to file 2024-11-27T12:56:29,740 DEBUG [RS:2;a3dd13ba50c2:40641 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T12:56:29,740 DEBUG [RS:2;a3dd13ba50c2:40641 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T12:56:29,741 INFO [regionserver/a3dd13ba50c2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:56:29,743 DEBUG [RS:2;a3dd13ba50c2:40641 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T12:56:29,743 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/default/testBlockEvictionOnGracefulStop1732712186733/26ce3555746db7891e983f64e97d670a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-27T12:56:29,743 DEBUG [RS:2;a3dd13ba50c2:40641 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732712189739 2024-11-27T12:56:29,743 DEBUG [RS:2;a3dd13ba50c2:40641 {}] bucket.BucketCache(1509): Thread RS:2;a3dd13ba50c2:40641 finished persisting bucket cache to file, renaming 2024-11-27T12:56:29,743 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T12:56:29,743 INFO [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:29,743 INFO [RS:2;a3dd13ba50c2:40641 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T12:56:29,743 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 26ce3555746db7891e983f64e97d670a: Waiting for close lock at 1732712189734Running coprocessor pre-close hooks at 1732712189734Disabling compacts and flushes for region at 1732712189734Disabling writes for close at 1732712189734Writing region close event to WAL at 1732712189736 (+2 ms)Running coprocessor post-close hooks at 1732712189743 (+7 ms)Closed at 1732712189743 2024-11-27T12:56:29,743 INFO [RS:2;a3dd13ba50c2:40641 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T12:56:29,743 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer(959): stopping server a3dd13ba50c2,40641,1732712188951 2024-11-27T12:56:29,743 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:56:29,743 DEBUG [RS_CLOSE_REGION-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a. 2024-11-27T12:56:29,744 INFO [RS:2;a3dd13ba50c2:40641 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;a3dd13ba50c2:40641. 2024-11-27T12:56:29,744 DEBUG [RS:2;a3dd13ba50c2:40641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:56:29,744 DEBUG [RS:2;a3dd13ba50c2:40641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:29,744 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer(976): stopping server a3dd13ba50c2,40641,1732712188951; all regions closed. 2024-11-27T12:56:29,751 DEBUG [RS:2;a3dd13ba50c2:40641 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs 2024-11-27T12:56:29,752 INFO [RS:2;a3dd13ba50c2:40641 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3dd13ba50c2%2C40641%2C1732712188951:(num 1732712189281) 2024-11-27T12:56:29,752 DEBUG [RS:2;a3dd13ba50c2:40641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:29,752 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:56:29,752 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:56:29,752 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.ChoreService(370): Chore service for: regionserver/a3dd13ba50c2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T12:56:29,752 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T12:56:29,752 INFO [regionserver/a3dd13ba50c2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:56:29,752 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T12:56:29,753 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T12:56:29,753 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:56:29,753 INFO [RS:2;a3dd13ba50c2:40641 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40641 2024-11-27T12:56:29,754 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9b36b4c260984e62b54120d45bc10328 2024-11-27T12:56:29,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:56:29,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40641-0x1017da74fca0003, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a3dd13ba50c2,40641,1732712188951 2024-11-27T12:56:29,767 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:56:29,768 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a3dd13ba50c2,40641,1732712188951] 2024-11-27T12:56:29,769 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/info/9b36b4c260984e62b54120d45bc10328 is 209, key is testBlockEvictionOnGracefulStop1732712186733,,1732712186749.26ce3555746db7891e983f64e97d670a./info:regioninfo/1732712189591/Put/seqid=0 2024-11-27T12:56:29,770 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/info/9b36b4c260984e62b54120d45bc10328 2024-11-27T12:56:29,772 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/.tmp/info/9b36b4c260984e62b54120d45bc10328 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/9b36b4c260984e62b54120d45bc10328 2024-11-27T12:56:29,774 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/9b36b4c260984e62b54120d45bc10328, delay=916 ms 2024-11-27T12:56:29,775 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/9b36b4c260984e62b54120d45bc10328, entries=8, sequenceid=19, filesize=6.6 K 2024-11-27T12:56:29,776 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 40ms, sequenceid=19, compaction requested=false 2024-11-27T12:56:29,776 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/8e4cfb98f131438b95443aefa379d930 2024-11-27T12:56:29,777 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/info/9b36b4c260984e62b54120d45bc10328 2024-11-27T12:56:29,777 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/ns/769b1265b78c48e2a37972e923fe607a 2024-11-27T12:56:29,778 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/table/69774ee77b4a408b98802d994f51c6ab 2024-11-27T12:56:29,783 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-27T12:56:29,783 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a3dd13ba50c2,40641,1732712188951 already deleted, retry=false 2024-11-27T12:56:29,783 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T12:56:29,783 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a3dd13ba50c2,40641,1732712188951 expired; onlineServers=1 2024-11-27T12:56:29,783 INFO [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T12:56:29,783 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732712189735Running coprocessor pre-close hooks at 1732712189735Disabling compacts and flushes for region at 1732712189735Disabling writes for close at 1732712189735Obtaining lock to block concurrent updates at 1732712189735Preparing flush snapshotting stores in 1588230740 at 1732712189735Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732712189736 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732712189736Flushing 1588230740/info: creating writer at 1732712189737 (+1 ms)Flushing 1588230740/info: appending metadata at 1732712189768 (+31 ms)Flushing 1588230740/info: closing flushed file at 1732712189769 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77274354: reopening flushed file at 1732712189772 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 40ms, sequenceid=19, compaction requested=false at 1732712189776 (+4 ms)Writing region close event to WAL at 1732712189778 (+2 ms)Running coprocessor post-close hooks at 1732712189783 (+5 ms)Closed at 1732712189783 2024-11-27T12:56:29,784 DEBUG [RS_CLOSE_META-regionserver/a3dd13ba50c2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T12:56:29,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40641-0x1017da74fca0003, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:29,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40641-0x1017da74fca0003, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:29,876 INFO [RS:2;a3dd13ba50c2:40641 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:56:29,876 INFO [RS:2;a3dd13ba50c2:40641 {}] regionserver.HRegionServer(1031): Exiting; stopping=a3dd13ba50c2,40641,1732712188951; zookeeper connection closed. 2024-11-27T12:56:29,877 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21cc5fe9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21cc5fe9 2024-11-27T12:56:29,935 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(976): stopping server a3dd13ba50c2,41341,1732712185256; all regions closed. 2024-11-27T12:56:29,937 DEBUG [RS:0;a3dd13ba50c2:41341 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs 2024-11-27T12:56:29,937 INFO [RS:0;a3dd13ba50c2:41341 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3dd13ba50c2%2C41341%2C1732712185256.meta:.meta(num 1732712189072) 2024-11-27T12:56:29,940 DEBUG [RS:0;a3dd13ba50c2:41341 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/oldWALs 2024-11-27T12:56:29,940 INFO [RS:0;a3dd13ba50c2:41341 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3dd13ba50c2%2C41341%2C1732712185256:(num 1732712186176) 2024-11-27T12:56:29,940 DEBUG [RS:0;a3dd13ba50c2:41341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:56:29,940 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:56:29,940 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:56:29,941 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.ChoreService(370): Chore service for: regionserver/a3dd13ba50c2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T12:56:29,941 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:56:29,941 INFO [regionserver/a3dd13ba50c2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:56:29,941 INFO [RS:0;a3dd13ba50c2:41341 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41341 2024-11-27T12:56:29,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a3dd13ba50c2,41341,1732712185256 2024-11-27T12:56:29,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:56:29,971 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:56:29,980 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a3dd13ba50c2,41341,1732712185256] 2024-11-27T12:56:29,988 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a3dd13ba50c2,41341,1732712185256 already deleted, retry=false 2024-11-27T12:56:29,988 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a3dd13ba50c2,41341,1732712185256 expired; onlineServers=0 2024-11-27T12:56:29,988 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a3dd13ba50c2,43841,1732712185108' ***** 2024-11-27T12:56:29,988 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T12:56:29,988 INFO [M:0;a3dd13ba50c2:43841 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:56:29,988 INFO [M:0;a3dd13ba50c2:43841 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:56:29,988 DEBUG [M:0;a3dd13ba50c2:43841 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T12:56:29,988 DEBUG [M:0;a3dd13ba50c2:43841 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T12:56:29,988 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.large.0-1732712185843 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.large.0-1732712185843,5,FailOnTimeoutGroup] 2024-11-27T12:56:29,988 DEBUG [master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.small.0-1732712185843 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3dd13ba50c2:0:becomeActiveMaster-HFileCleaner.small.0-1732712185843,5,FailOnTimeoutGroup] 2024-11-27T12:56:29,988 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T12:56:29,989 INFO [M:0;a3dd13ba50c2:43841 {}] hbase.ChoreService(370): Chore service for: master/a3dd13ba50c2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T12:56:29,989 INFO [M:0;a3dd13ba50c2:43841 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:56:29,989 DEBUG [M:0;a3dd13ba50c2:43841 {}] master.HMaster(1795): Stopping service threads 2024-11-27T12:56:29,989 INFO [M:0;a3dd13ba50c2:43841 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T12:56:29,989 INFO [M:0;a3dd13ba50c2:43841 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T12:56:29,989 INFO [M:0;a3dd13ba50c2:43841 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T12:56:29,989 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T12:56:30,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T12:56:30,000 DEBUG [M:0;a3dd13ba50c2:43841 {}] zookeeper.ZKUtil(347): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T12:56:30,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:56:30,000 WARN [M:0;a3dd13ba50c2:43841 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T12:56:30,000 INFO [M:0;a3dd13ba50c2:43841 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/.lastflushedseqids 2024-11-27T12:56:30,006 INFO [M:0;a3dd13ba50c2:43841 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T12:56:30,006 INFO [M:0;a3dd13ba50c2:43841 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T12:56:30,007 DEBUG [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T12:56:30,007 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:30,007 DEBUG [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:30,007 DEBUG [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T12:56:30,007 DEBUG [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:30,007 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.49 KB heapSize=49.88 KB 2024-11-27T12:56:30,016 DEBUG [M:0;a3dd13ba50c2:43841 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9b527a57341a47a6a37e1564a69238f6 2024-11-27T12:56:30,030 DEBUG [M:0;a3dd13ba50c2:43841 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b527a57341a47a6a37e1564a69238f6 is 82, key is hbase:meta,,1/info:regioninfo/1732712189133/Put/seqid=0 2024-11-27T12:56:30,031 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b527a57341a47a6a37e1564a69238f6 2024-11-27T12:56:30,039 DEBUG [M:0;a3dd13ba50c2:43841 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c2bd28d7426049dc82c946c5a12cc8ac 2024-11-27T12:56:30,058 DEBUG [M:0;a3dd13ba50c2:43841 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2bd28d7426049dc82c946c5a12cc8ac is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732712187199/Put/seqid=0 2024-11-27T12:56:30,058 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.26 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2bd28d7426049dc82c946c5a12cc8ac 2024-11-27T12:56:30,066 DEBUG [M:0;a3dd13ba50c2:43841 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ffbf45fd558d4c348eaff23f1f47829d 2024-11-27T12:56:30,079 DEBUG [M:0;a3dd13ba50c2:43841 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ffbf45fd558d4c348eaff23f1f47829d is 69, key is a3dd13ba50c2,40641,1732712188951/rs:state/1732712189100/Put/seqid=0 2024-11-27T12:56:30,080 INFO [RS:0;a3dd13ba50c2:41341 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:56:30,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:30,080 INFO [RS:0;a3dd13ba50c2:41341 {}] regionserver.HRegionServer(1031): Exiting; stopping=a3dd13ba50c2,41341,1732712185256; zookeeper connection closed. 2024-11-27T12:56:30,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41341-0x1017da74fca0001, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:30,080 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ffbf45fd558d4c348eaff23f1f47829d 2024-11-27T12:56:30,080 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a529dae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a529dae 2024-11-27T12:56:30,081 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-27T12:56:30,082 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ffbf45fd558d4c348eaff23f1f47829d 2024-11-27T12:56:30,082 DEBUG [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b527a57341a47a6a37e1564a69238f6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9b527a57341a47a6a37e1564a69238f6 2024-11-27T12:56:30,084 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9b527a57341a47a6a37e1564a69238f6, entries=14, sequenceid=102, filesize=5.9 K 2024-11-27T12:56:30,085 DEBUG [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2bd28d7426049dc82c946c5a12cc8ac as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2bd28d7426049dc82c946c5a12cc8ac 2024-11-27T12:56:30,086 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2bd28d7426049dc82c946c5a12cc8ac, entries=11, sequenceid=102, filesize=6.5 K 2024-11-27T12:56:30,087 DEBUG [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ffbf45fd558d4c348eaff23f1f47829d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ffbf45fd558d4c348eaff23f1f47829d 2024-11-27T12:56:30,089 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ffbf45fd558d4c348eaff23f1f47829d 2024-11-27T12:56:30,089 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ee70cdda-f4f8-9f27-8050-98a06dbf239c/d35a4daf-47ed-9ba4-05ae-b98d09cf5ed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ffbf45fd558d4c348eaff23f1f47829d, entries=3, sequenceid=102, filesize=5.3 K 2024-11-27T12:56:30,090 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=102, compaction requested=false 2024-11-27T12:56:30,095 INFO [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:56:30,095 DEBUG [M:0;a3dd13ba50c2:43841 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732712190007Disabling compacts and flushes for region at 1732712190007Disabling writes for close at 1732712190007Obtaining lock to block concurrent updates at 1732712190007Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732712190007Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40436, getHeapSize=51016, getOffHeapSize=0, getCellsCount=127 at 1732712190008 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732712190008Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732712190008Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732712190030 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732712190030Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732712190033 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732712190057 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732712190057Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732712190060 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732712190078 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732712190078Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69d2ec4f: reopening flushed file at 1732712190082 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7aaa3fcd: reopening flushed file at 1732712190084 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35a78131: reopening flushed file at 1732712190086 (+2 ms)Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=102, compaction requested=false at 1732712190090 (+4 ms)Writing region close event to WAL at 1732712190095 (+5 ms)Closed at 1732712190095 2024-11-27T12:56:30,096 INFO [M:0;a3dd13ba50c2:43841 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T12:56:30,096 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:56:30,096 INFO [M:0;a3dd13ba50c2:43841 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43841 2024-11-27T12:56:30,097 INFO [M:0;a3dd13ba50c2:43841 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:56:30,217 INFO [M:0;a3dd13ba50c2:43841 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:56:30,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:30,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43841-0x1017da74fca0000, quorum=127.0.0.1:52261, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:56:30,219 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T12:56:30,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-27T12:56:30,220 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:56:30,232 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 52) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=486 (was 467) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9033 (was 9096)