2025-01-11 14:38:16,357 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2025-01-11 14:38:16,377 main DEBUG Took 0.017872 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2025-01-11 14:38:16,377 main DEBUG PluginManager 'Core' found 129 plugins 2025-01-11 14:38:16,377 main DEBUG PluginManager 'Level' found 0 plugins 2025-01-11 14:38:16,378 main DEBUG PluginManager 'Lookup' found 16 plugins 2025-01-11 14:38:16,380 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,386 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2025-01-11 14:38:16,398 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,400 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,400 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,400 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,401 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,401 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,402 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,402 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,403 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,403 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,404 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,404 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,404 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,405 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,405 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,406 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,406 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,407 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,407 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,408 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,408 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,409 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,409 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,410 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-11 14:38:16,410 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,411 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2025-01-11 14:38:16,412 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-11 14:38:16,414 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2025-01-11 14:38:16,416 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2025-01-11 14:38:16,416 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2025-01-11 14:38:16,418 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2025-01-11 14:38:16,418 main DEBUG PluginManager 'Converter' found 47 plugins 2025-01-11 14:38:16,430 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2025-01-11 14:38:16,433 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2025-01-11 14:38:16,434 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2025-01-11 14:38:16,435 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2025-01-11 14:38:16,435 main DEBUG createAppenders(={Console}) 2025-01-11 14:38:16,436 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2025-01-11 14:38:16,436 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2025-01-11 14:38:16,437 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2025-01-11 14:38:16,437 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2025-01-11 14:38:16,438 main DEBUG OutputStream closed 2025-01-11 14:38:16,438 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2025-01-11 14:38:16,439 main DEBUG Appender DefaultConsole-1 stopped with status true 2025-01-11 14:38:16,439 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2025-01-11 14:38:16,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2025-01-11 14:38:16,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2025-01-11 14:38:16,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2025-01-11 14:38:16,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2025-01-11 14:38:16,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2025-01-11 14:38:16,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2025-01-11 14:38:16,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2025-01-11 14:38:16,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2025-01-11 14:38:16,524 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2025-01-11 14:38:16,524 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2025-01-11 14:38:16,524 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2025-01-11 14:38:16,524 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2025-01-11 14:38:16,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2025-01-11 14:38:16,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2025-01-11 14:38:16,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2025-01-11 14:38:16,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2025-01-11 14:38:16,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2025-01-11 14:38:16,527 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2025-01-11 14:38:16,529 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2025-01-11 14:38:16,529 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2025-01-11 14:38:16,530 main DEBUG Shutdown hook enabled. Registering a new one. 2025-01-11 14:38:16,531 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2025-01-11T14:38:16,550 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2025-01-11 14:38:16,553 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2025-01-11 14:38:16,553 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2025-01-11T14:38:16,848 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0 2025-01-11T14:38:16,878 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=189, ProcessCount=11, AvailableMemoryMB=4368 2025-01-11T14:38:16,996 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2025-01-11T14:38:17,097 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:17,114 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/cluster_909f1d1e-4017-7f4f-f9f9-8e75d6bae4f8, deleteOnExit=true 2025-01-11T14:38:17,226 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/cluster_909f1d1e-4017-7f4f-f9f9-8e75d6bae4f8/zookeeper_0, clientPort=58962, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/cluster_909f1d1e-4017-7f4f-f9f9-8e75d6bae4f8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/cluster_909f1d1e-4017-7f4f-f9f9-8e75d6bae4f8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2025-01-11T14:38:17,239 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58962 2025-01-11T14:38:17,239 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:17,324 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a with version=8 2025-01-11T14:38:17,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/hbase-staging 2025-01-11T14:38:17,429 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2025-01-11T14:38:17,727 INFO [Time-limited test {}] client.ConnectionUtils(128): master/df3d9de72c82:0 server-side Connection retries=45 2025-01-11T14:38:17,738 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:17,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:17,747 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2025-01-11T14:38:17,747 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:17,747 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2025-01-11T14:38:17,901 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2025-01-11T14:38:17,992 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2025-01-11T14:38:18,029 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2025-01-11T14:38:18,040 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2025-01-11T14:38:18,044 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-11T14:38:18,087 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 514 (auto-detected) 2025-01-11T14:38:18,088 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2025-01-11T14:38:18,109 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37769 2025-01-11T14:38:18,133 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37769 connecting to ZooKeeper ensemble=127.0.0.1:58962 2025-01-11T14:38:18,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:377690x0, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2025-01-11T14:38:18,172 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37769-0x101696b37120000 connected 2025-01-11T14:38:18,199 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:18,200 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:18,208 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:18,212 INFO [Time-limited test {}] master.HMaster(536): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a, hbase.cluster.distributed=false 2025-01-11T14:38:18,236 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2025-01-11T14:38:18,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37769 2025-01-11T14:38:18,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37769 2025-01-11T14:38:18,243 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37769 2025-01-11T14:38:18,243 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37769 2025-01-11T14:38:18,244 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37769 2025-01-11T14:38:18,343 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/df3d9de72c82:0 server-side Connection retries=45 2025-01-11T14:38:18,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:18,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:18,346 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2025-01-11T14:38:18,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:18,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2025-01-11T14:38:18,351 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2025-01-11T14:38:18,354 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-11T14:38:18,355 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33459 2025-01-11T14:38:18,357 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33459 connecting to ZooKeeper ensemble=127.0.0.1:58962 2025-01-11T14:38:18,357 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:18,358 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:18,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334590x0, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2025-01-11T14:38:18,363 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33459-0x101696b37120001 connected 2025-01-11T14:38:18,363 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:18,371 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2025-01-11T14:38:18,383 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/bucket.cache 2025-01-11T14:38:18,389 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2025-01-11T14:38:18,390 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2025-01-11T14:38:18,392 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2025-01-11T14:38:18,394 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2025-01-11T14:38:18,394 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-11T14:38:18,402 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-11T14:38:18,406 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-11T14:38:18,406 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/bucket.persistence is complete. 2025-01-11T14:38:18,408 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2025-01-11T14:38:18,411 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2025-01-11T14:38:18,417 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2025-01-11T14:38:18,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33459 2025-01-11T14:38:18,419 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33459 2025-01-11T14:38:18,421 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33459 2025-01-11T14:38:18,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33459 2025-01-11T14:38:18,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33459 2025-01-11T14:38:18,446 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/df3d9de72c82:0 server-side Connection retries=45 2025-01-11T14:38:18,446 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:18,446 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:18,447 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2025-01-11T14:38:18,447 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:18,447 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2025-01-11T14:38:18,447 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2025-01-11T14:38:18,447 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-11T14:38:18,449 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38289 2025-01-11T14:38:18,450 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38289 connecting to ZooKeeper ensemble=127.0.0.1:58962 2025-01-11T14:38:18,451 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:18,452 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:18,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382890x0, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2025-01-11T14:38:18,457 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38289-0x101696b37120002 connected 2025-01-11T14:38:18,457 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:18,458 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2025-01-11T14:38:18,460 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/bucket.cache 2025-01-11T14:38:18,460 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2025-01-11T14:38:18,461 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2025-01-11T14:38:18,461 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2025-01-11T14:38:18,462 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2025-01-11T14:38:18,462 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-11T14:38:18,462 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-11T14:38:18,463 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-11T14:38:18,463 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/bucket.persistence is complete. 2025-01-11T14:38:18,464 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2025-01-11T14:38:18,466 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2025-01-11T14:38:18,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2025-01-11T14:38:18,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38289 2025-01-11T14:38:18,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38289 2025-01-11T14:38:18,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38289 2025-01-11T14:38:18,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38289 2025-01-11T14:38:18,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38289 2025-01-11T14:38:18,498 DEBUG [M:0;df3d9de72c82:37769 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;df3d9de72c82:37769 2025-01-11T14:38:18,499 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(2525): Adding backup master ZNode /hbase/backup-masters/df3d9de72c82,37769,1736606297517 2025-01-11T14:38:18,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:18,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:18,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:18,509 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/df3d9de72c82,37769,1736606297517 2025-01-11T14:38:18,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2025-01-11T14:38:18,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2025-01-11T14:38:18,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:18,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:18,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:18,537 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2025-01-11T14:38:18,538 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/df3d9de72c82,37769,1736606297517 from backup master directory 2025-01-11T14:38:18,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:18,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/df3d9de72c82,37769,1736606297517 2025-01-11T14:38:18,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:18,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:18,543 WARN [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2025-01-11T14:38:18,543 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=df3d9de72c82,37769,1736606297517 2025-01-11T14:38:18,545 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2025-01-11T14:38:18,548 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2025-01-11T14:38:18,562 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/hbase.id] with ID: 708b124d-3a3d-4cb0-a16c-f3e3b764637c 2025-01-11T14:38:18,563 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/.tmp/hbase.id 2025-01-11T14:38:18,571 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/hbase.id] 2025-01-11T14:38:18,620 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:18,625 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2025-01-11T14:38:18,647 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2025-01-11T14:38:18,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:18,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:18,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:18,686 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2025-01-11T14:38:18,689 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2025-01-11T14:38:18,736 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(245): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:184) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:243) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1014) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:624) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-11T14:38:18,744 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2025-01-11T14:38:18,808 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store 2025-01-11T14:38:18,834 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2025-01-11T14:38:18,838 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:18,840 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2025-01-11T14:38:18,840 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:18,840 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:18,843 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2025-01-11T14:38:18,843 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:18,843 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:18,844 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1736606298840Disabling compacts and flushes for region at 1736606298840Disabling writes for close at 1736606298843 (+3 ms)Writing region close event to WAL at 1736606298843Closed at 1736606298843 2025-01-11T14:38:18,845 WARN [master/df3d9de72c82:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/.initializing 2025-01-11T14:38:18,845 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/WALs/df3d9de72c82,37769,1736606297517 2025-01-11T14:38:18,860 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2025-01-11T14:38:18,877 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=df3d9de72c82%2C37769%2C1736606297517, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/WALs/df3d9de72c82,37769,1736606297517, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/oldWALs, maxLogs=10 2025-01-11T14:38:18,921 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/WALs/df3d9de72c82,37769,1736606297517/df3d9de72c82%2C37769%2C1736606297517.1736606298884 2025-01-11T14:38:18,922 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2025-01-11T14:38:18,923 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2025-01-11T14:38:18,923 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:18,927 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:18,928 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:18,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:19,008 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2025-01-11T14:38:19,013 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:19,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:19,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:19,021 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2025-01-11T14:38:19,021 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:19,021 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2025-01-11T14:38:19,022 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:19,027 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2025-01-11T14:38:19,027 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:19,028 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2025-01-11T14:38:19,028 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:19,034 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2025-01-11T14:38:19,034 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:19,035 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2025-01-11T14:38:19,035 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:19,039 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:19,039 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:19,042 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:19,043 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:19,046 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2025-01-11T14:38:19,048 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:19,057 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2025-01-11T14:38:19,058 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67109430, jitterRate=8.434057235717773E-6}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2025-01-11T14:38:19,065 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1736606298941Initializing all the Stores at 1736606298941Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606298942 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606298943 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606298944 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606298944Cleaning up temporary data from old regions at 1736606299043 (+99 ms)Region opened successfully at 1736606299064 (+21 ms) 2025-01-11T14:38:19,066 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2025-01-11T14:38:19,101 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c4547cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=df3d9de72c82/172.17.0.2:0 2025-01-11T14:38:19,136 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(923): No meta location available on zookeeper, skip migrating... 2025-01-11T14:38:19,149 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2025-01-11T14:38:19,149 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2025-01-11T14:38:19,153 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2025-01-11T14:38:19,153 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2025-01-11T14:38:19,160 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2025-01-11T14:38:19,160 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2025-01-11T14:38:19,193 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2025-01-11T14:38:19,203 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2025-01-11T14:38:19,206 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2025-01-11T14:38:19,209 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2025-01-11T14:38:19,210 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2025-01-11T14:38:19,212 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2025-01-11T14:38:19,215 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2025-01-11T14:38:19,219 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2025-01-11T14:38:19,221 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2025-01-11T14:38:19,222 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2025-01-11T14:38:19,224 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2025-01-11T14:38:19,243 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2025-01-11T14:38:19,244 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2025-01-11T14:38:19,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:19,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:19,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:19,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:19,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:19,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:19,261 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(867): Active/primary master=df3d9de72c82,37769,1736606297517, sessionid=0x101696b37120000, setting cluster-up flag (Was=false) 2025-01-11T14:38:19,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:19,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:19,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:19,280 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2025-01-11T14:38:19,283 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=df3d9de72c82,37769,1736606297517 2025-01-11T14:38:19,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:19,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:19,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:19,292 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2025-01-11T14:38:19,294 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=df3d9de72c82,37769,1736606297517 2025-01-11T14:38:19,299 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2025-01-11T14:38:19,376 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2025-01-11T14:38:19,385 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer(746): ClusterId : 708b124d-3a3d-4cb0-a16c-f3e3b764637c 2025-01-11T14:38:19,385 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(746): ClusterId : 708b124d-3a3d-4cb0-a16c-f3e3b764637c 2025-01-11T14:38:19,388 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2025-01-11T14:38:19,388 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2025-01-11T14:38:19,389 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2025-01-11T14:38:19,394 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2025-01-11T14:38:19,394 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2025-01-11T14:38:19,394 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2025-01-11T14:38:19,394 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2025-01-11T14:38:19,398 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2025-01-11T14:38:19,398 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2025-01-11T14:38:19,398 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2025-01-11T14:38:19,399 DEBUG [RS:1;df3d9de72c82:38289 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f26f357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=df3d9de72c82/172.17.0.2:0 2025-01-11T14:38:19,399 DEBUG [RS:0;df3d9de72c82:33459 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f358b3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=df3d9de72c82/172.17.0.2:0 2025-01-11T14:38:19,405 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: df3d9de72c82,37769,1736606297517 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2025-01-11T14:38:19,415 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/df3d9de72c82:0, corePoolSize=5, maxPoolSize=5 2025-01-11T14:38:19,416 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/df3d9de72c82:0, corePoolSize=5, maxPoolSize=5 2025-01-11T14:38:19,416 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/df3d9de72c82:0, corePoolSize=5, maxPoolSize=5 2025-01-11T14:38:19,416 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/df3d9de72c82:0, corePoolSize=5, maxPoolSize=5 2025-01-11T14:38:19,417 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/df3d9de72c82:0, corePoolSize=10, maxPoolSize=10 2025-01-11T14:38:19,417 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,417 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/df3d9de72c82:0, corePoolSize=2, maxPoolSize=2 2025-01-11T14:38:19,417 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,420 DEBUG [RS:0;df3d9de72c82:33459 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;df3d9de72c82:33459 2025-01-11T14:38:19,421 DEBUG [RS:1;df3d9de72c82:38289 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;df3d9de72c82:38289 2025-01-11T14:38:19,425 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2025-01-11T14:38:19,425 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2025-01-11T14:38:19,425 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2025-01-11T14:38:19,425 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2025-01-11T14:38:19,425 DEBUG [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer(832): About to register with Master. 2025-01-11T14:38:19,425 DEBUG [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(832): About to register with Master. 2025-01-11T14:38:19,428 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2025-01-11T14:38:19,429 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer(2659): reportForDuty to master=df3d9de72c82,37769,1736606297517 with port=33459, startcode=1736606298299 2025-01-11T14:38:19,429 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(2659): reportForDuty to master=df3d9de72c82,37769,1736606297517 with port=38289, startcode=1736606298445 2025-01-11T14:38:19,429 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2025-01-11T14:38:19,430 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1736606329430 2025-01-11T14:38:19,432 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2025-01-11T14:38:19,434 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2025-01-11T14:38:19,435 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:19,436 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2025-01-11T14:38:19,438 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2025-01-11T14:38:19,439 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2025-01-11T14:38:19,439 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2025-01-11T14:38:19,439 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2025-01-11T14:38:19,441 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,446 DEBUG [RS:1;df3d9de72c82:38289 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2025-01-11T14:38:19,446 DEBUG [RS:0;df3d9de72c82:33459 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2025-01-11T14:38:19,447 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2025-01-11T14:38:19,449 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2025-01-11T14:38:19,449 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2025-01-11T14:38:19,454 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2025-01-11T14:38:19,455 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2025-01-11T14:38:19,458 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.large.0-1736606299456,5,FailOnTimeoutGroup] 2025-01-11T14:38:19,468 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.small.0-1736606299458,5,FailOnTimeoutGroup] 2025-01-11T14:38:19,469 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,469 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(1752): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2025-01-11T14:38:19,470 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,470 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,484 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2025-01-11T14:38:19,485 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a 2025-01-11T14:38:19,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:19,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2025-01-11T14:38:19,511 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2025-01-11T14:38:19,511 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:19,511 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:19,512 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2025-01-11T14:38:19,517 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2025-01-11T14:38:19,517 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:19,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:19,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2025-01-11T14:38:19,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2025-01-11T14:38:19,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:19,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:19,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2025-01-11T14:38:19,531 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37009, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2025-01-11T14:38:19,531 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35975, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2025-01-11T14:38:19,533 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2025-01-11T14:38:19,533 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:19,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:19,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2025-01-11T14:38:19,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740 2025-01-11T14:38:19,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740 2025-01-11T14:38:19,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2025-01-11T14:38:19,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2025-01-11T14:38:19,536 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2025-01-11T14:38:19,537 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2025-01-11T14:38:19,539 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37769 {}] master.ServerManager(363): Checking decommissioned status of RegionServer df3d9de72c82,33459,1736606298299 2025-01-11T14:38:19,543 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37769 {}] master.ServerManager(517): Registering regionserver=df3d9de72c82,33459,1736606298299 2025-01-11T14:38:19,547 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2025-01-11T14:38:19,548 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73307172, jitterRate=0.09236198663711548}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2025-01-11T14:38:19,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1736606299501Initializing all the Stores at 1736606299502 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606299502Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606299502Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606299502Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606299502Cleaning up temporary data from old regions at 1736606299536 (+34 ms)Region opened successfully at 1736606299551 (+15 ms) 2025-01-11T14:38:19,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2025-01-11T14:38:19,551 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2025-01-11T14:38:19,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2025-01-11T14:38:19,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2025-01-11T14:38:19,552 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2025-01-11T14:38:19,553 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2025-01-11T14:38:19,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1736606299551Disabling compacts and flushes for region at 1736606299551Disabling writes for close at 1736606299552 (+1 ms)Writing region close event to WAL at 1736606299553 (+1 ms)Closed at 1736606299553 2025-01-11T14:38:19,556 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2025-01-11T14:38:19,557 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2025-01-11T14:38:19,557 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37769 {}] master.ServerManager(363): Checking decommissioned status of RegionServer df3d9de72c82,38289,1736606298445 2025-01-11T14:38:19,558 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37769 {}] master.ServerManager(517): Registering regionserver=df3d9de72c82,38289,1736606298445 2025-01-11T14:38:19,561 DEBUG [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a 2025-01-11T14:38:19,561 DEBUG [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a 2025-01-11T14:38:19,561 DEBUG [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2025-01-11T14:38:19,561 DEBUG [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2025-01-11T14:38:19,561 DEBUG [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2025-01-11T14:38:19,561 DEBUG [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2025-01-11T14:38:19,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2025-01-11T14:38:19,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2025-01-11T14:38:19,569 DEBUG [RS:1;df3d9de72c82:38289 {}] zookeeper.ZKUtil(111): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/df3d9de72c82,38289,1736606298445 2025-01-11T14:38:19,569 WARN [RS:1;df3d9de72c82:38289 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2025-01-11T14:38:19,570 DEBUG [RS:0;df3d9de72c82:33459 {}] zookeeper.ZKUtil(111): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/df3d9de72c82,33459,1736606298299 2025-01-11T14:38:19,570 INFO [RS:1;df3d9de72c82:38289 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2025-01-11T14:38:19,570 WARN [RS:0;df3d9de72c82:33459 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2025-01-11T14:38:19,570 INFO [RS:0;df3d9de72c82:33459 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2025-01-11T14:38:19,570 DEBUG [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,38289,1736606298445 2025-01-11T14:38:19,570 DEBUG [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,33459,1736606298299 2025-01-11T14:38:19,573 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [df3d9de72c82,33459,1736606298299] 2025-01-11T14:38:19,573 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [df3d9de72c82,38289,1736606298445] 2025-01-11T14:38:19,575 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2025-01-11T14:38:19,577 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2025-01-11T14:38:19,604 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2025-01-11T14:38:19,604 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2025-01-11T14:38:19,623 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2025-01-11T14:38:19,625 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2025-01-11T14:38:19,632 INFO [RS:0;df3d9de72c82:33459 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2025-01-11T14:38:19,632 INFO [RS:1;df3d9de72c82:38289 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2025-01-11T14:38:19,632 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,632 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,633 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2025-01-11T14:38:19,639 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2025-01-11T14:38:19,639 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2025-01-11T14:38:19,640 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2025-01-11T14:38:19,640 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,640 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,641 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,641 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,641 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,641 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,641 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,641 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,641 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,641 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,641 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,641 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/df3d9de72c82:0, corePoolSize=2, maxPoolSize=2 2025-01-11T14:38:19,642 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,642 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,642 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,642 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/df3d9de72c82:0, corePoolSize=2, maxPoolSize=2 2025-01-11T14:38:19,642 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,642 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,642 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,642 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,642 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,642 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,642 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/df3d9de72c82:0, corePoolSize=3, maxPoolSize=3 2025-01-11T14:38:19,642 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,643 DEBUG [RS:0;df3d9de72c82:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/df3d9de72c82:0, corePoolSize=3, maxPoolSize=3 2025-01-11T14:38:19,643 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,643 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,643 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:19,643 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/df3d9de72c82:0, corePoolSize=3, maxPoolSize=3 2025-01-11T14:38:19,644 DEBUG [RS:1;df3d9de72c82:38289 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/df3d9de72c82:0, corePoolSize=3, maxPoolSize=3 2025-01-11T14:38:19,645 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,645 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,645 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,645 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,645 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,645 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,33459,1736606298299-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2025-01-11T14:38:19,651 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,651 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,651 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,651 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,651 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,652 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,38289,1736606298445-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2025-01-11T14:38:19,671 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2025-01-11T14:38:19,673 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,33459,1736606298299-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,674 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,674 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.Replication(171): df3d9de72c82,33459,1736606298299 started 2025-01-11T14:38:19,678 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2025-01-11T14:38:19,678 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,38289,1736606298445-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,678 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,678 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.Replication(171): df3d9de72c82,38289,1736606298445 started 2025-01-11T14:38:19,694 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,694 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer(1482): Serving as df3d9de72c82,33459,1736606298299, RpcServer on df3d9de72c82/172.17.0.2:33459, sessionid=0x101696b37120001 2025-01-11T14:38:19,695 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2025-01-11T14:38:19,695 DEBUG [RS:0;df3d9de72c82:33459 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager df3d9de72c82,33459,1736606298299 2025-01-11T14:38:19,696 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'df3d9de72c82,33459,1736606298299' 2025-01-11T14:38:19,696 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2025-01-11T14:38:19,697 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2025-01-11T14:38:19,698 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2025-01-11T14:38:19,698 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2025-01-11T14:38:19,698 DEBUG [RS:0;df3d9de72c82:33459 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager df3d9de72c82,33459,1736606298299 2025-01-11T14:38:19,698 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'df3d9de72c82,33459,1736606298299' 2025-01-11T14:38:19,698 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2025-01-11T14:38:19,699 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2025-01-11T14:38:19,699 DEBUG [RS:0;df3d9de72c82:33459 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2025-01-11T14:38:19,700 INFO [RS:0;df3d9de72c82:33459 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2025-01-11T14:38:19,700 INFO [RS:0;df3d9de72c82:33459 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2025-01-11T14:38:19,704 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:19,705 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(1482): Serving as df3d9de72c82,38289,1736606298445, RpcServer on df3d9de72c82/172.17.0.2:38289, sessionid=0x101696b37120002 2025-01-11T14:38:19,705 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2025-01-11T14:38:19,705 DEBUG [RS:1;df3d9de72c82:38289 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager df3d9de72c82,38289,1736606298445 2025-01-11T14:38:19,705 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'df3d9de72c82,38289,1736606298445' 2025-01-11T14:38:19,705 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2025-01-11T14:38:19,706 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2025-01-11T14:38:19,707 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2025-01-11T14:38:19,707 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2025-01-11T14:38:19,707 DEBUG [RS:1;df3d9de72c82:38289 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager df3d9de72c82,38289,1736606298445 2025-01-11T14:38:19,707 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'df3d9de72c82,38289,1736606298445' 2025-01-11T14:38:19,707 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2025-01-11T14:38:19,708 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2025-01-11T14:38:19,708 DEBUG [RS:1;df3d9de72c82:38289 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2025-01-11T14:38:19,708 INFO [RS:1;df3d9de72c82:38289 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2025-01-11T14:38:19,709 INFO [RS:1;df3d9de72c82:38289 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2025-01-11T14:38:19,729 WARN [df3d9de72c82:37769 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2025-01-11T14:38:19,806 INFO [RS:0;df3d9de72c82:33459 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2025-01-11T14:38:19,807 INFO [RS:0;df3d9de72c82:33459 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=df3d9de72c82%2C33459%2C1736606298299, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,33459,1736606298299, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs, maxLogs=55 2025-01-11T14:38:19,810 INFO [RS:1;df3d9de72c82:38289 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2025-01-11T14:38:19,810 INFO [RS:1;df3d9de72c82:38289 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=df3d9de72c82%2C38289%2C1736606298445, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,38289,1736606298445, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs, maxLogs=55 2025-01-11T14:38:19,832 INFO [RS:1;df3d9de72c82:38289 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,38289,1736606298445/df3d9de72c82%2C38289%2C1736606298445.1736606299812 2025-01-11T14:38:19,832 DEBUG [RS:1;df3d9de72c82:38289 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2025-01-11T14:38:19,833 INFO [RS:0;df3d9de72c82:33459 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,33459,1736606298299/df3d9de72c82%2C33459%2C1736606298299.1736606299810 2025-01-11T14:38:19,833 DEBUG [RS:0;df3d9de72c82:33459 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2025-01-11T14:38:19,981 DEBUG [df3d9de72c82:37769 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2025-01-11T14:38:19,993 DEBUG [df3d9de72c82:37769 {}] balancer.BalancerClusterState(204): Hosts are {df3d9de72c82=0} racks are {/default-rack=0} 2025-01-11T14:38:19,996 DEBUG [df3d9de72c82:37769 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2025-01-11T14:38:19,996 DEBUG [df3d9de72c82:37769 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2025-01-11T14:38:19,996 DEBUG [df3d9de72c82:37769 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2025-01-11T14:38:19,996 DEBUG [df3d9de72c82:37769 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2025-01-11T14:38:19,996 INFO [df3d9de72c82:37769 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2025-01-11T14:38:19,996 INFO [df3d9de72c82:37769 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2025-01-11T14:38:19,996 DEBUG [df3d9de72c82:37769 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2025-01-11T14:38:20,004 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=df3d9de72c82,38289,1736606298445 2025-01-11T14:38:20,010 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as df3d9de72c82,38289,1736606298445, state=OPENING 2025-01-11T14:38:20,015 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2025-01-11T14:38:20,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:20,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:20,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:20,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:20,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:20,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:20,020 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2025-01-11T14:38:20,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=df3d9de72c82,38289,1736606298445}] 2025-01-11T14:38:20,195 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2025-01-11T14:38:20,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50887, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2025-01-11T14:38:20,206 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2025-01-11T14:38:20,206 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2025-01-11T14:38:20,207 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2025-01-11T14:38:20,208 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=df3d9de72c82%2C38289%2C1736606298445.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,38289,1736606298445, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs, maxLogs=55 2025-01-11T14:38:20,230 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,38289,1736606298445/df3d9de72c82%2C38289%2C1736606298445.meta.1736606300210.meta 2025-01-11T14:38:20,230 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2025-01-11T14:38:20,231 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2025-01-11T14:38:20,233 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2025-01-11T14:38:20,235 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2025-01-11T14:38:20,240 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2025-01-11T14:38:20,244 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2025-01-11T14:38:20,245 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:20,245 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2025-01-11T14:38:20,245 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2025-01-11T14:38:20,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2025-01-11T14:38:20,248 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2025-01-11T14:38:20,248 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:20,248 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:20,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2025-01-11T14:38:20,249 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2025-01-11T14:38:20,249 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:20,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:20,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2025-01-11T14:38:20,251 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2025-01-11T14:38:20,251 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:20,251 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:20,251 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2025-01-11T14:38:20,252 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2025-01-11T14:38:20,252 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:20,252 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:20,253 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2025-01-11T14:38:20,254 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740 2025-01-11T14:38:20,254 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740 2025-01-11T14:38:20,255 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2025-01-11T14:38:20,255 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2025-01-11T14:38:20,255 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2025-01-11T14:38:20,256 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2025-01-11T14:38:20,257 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70737952, jitterRate=0.0540776252746582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2025-01-11T14:38:20,257 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2025-01-11T14:38:20,258 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1736606300246Writing region info on filesystem at 1736606300246Initializing all the Stores at 1736606300246Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606300246Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606300247 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606300247Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606300247Cleaning up temporary data from old regions at 1736606300255 (+8 ms)Running coprocessor post-open hooks at 1736606300257 (+2 ms)Region opened successfully at 1736606300258 (+1 ms) 2025-01-11T14:38:20,265 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1736606300186 2025-01-11T14:38:20,275 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2025-01-11T14:38:20,276 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2025-01-11T14:38:20,277 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=df3d9de72c82,38289,1736606298445 2025-01-11T14:38:20,278 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as df3d9de72c82,38289,1736606298445, state=OPEN 2025-01-11T14:38:20,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2025-01-11T14:38:20,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2025-01-11T14:38:20,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2025-01-11T14:38:20,281 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:20,281 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:20,281 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:20,282 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=df3d9de72c82,38289,1736606298445 2025-01-11T14:38:20,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2025-01-11T14:38:20,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=df3d9de72c82,38289,1736606298445 in 260 msec 2025-01-11T14:38:20,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2025-01-11T14:38:20,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 721 msec 2025-01-11T14:38:20,290 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2025-01-11T14:38:20,290 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2025-01-11T14:38:20,311 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2025-01-11T14:38:20,313 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=df3d9de72c82,38289,1736606298445, seqNum=-1] 2025-01-11T14:38:20,333 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2025-01-11T14:38:20,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42579, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2025-01-11T14:38:20,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0250 sec 2025-01-11T14:38:20,355 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(1134): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1736606300355, completionTime=-1 2025-01-11T14:38:20,358 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2025-01-11T14:38:20,358 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2025-01-11T14:38:20,387 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2025-01-11T14:38:20,387 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1736606360387 2025-01-11T14:38:20,387 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1736606420387 2025-01-11T14:38:20,388 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2025-01-11T14:38:20,390 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2025-01-11T14:38:20,396 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,37769,1736606297517-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:20,396 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,37769,1736606297517-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:20,396 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,37769,1736606297517-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:20,398 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-df3d9de72c82:37769, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:20,398 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:20,399 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:20,406 DEBUG [master/df3d9de72c82:0.Chore.1 {}] janitor.CatalogJanitor(180): 2025-01-11T14:38:20,427 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(1250): Master has completed initialization 1.884sec 2025-01-11T14:38:20,429 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2025-01-11T14:38:20,430 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2025-01-11T14:38:20,431 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2025-01-11T14:38:20,431 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2025-01-11T14:38:20,431 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2025-01-11T14:38:20,432 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,37769,1736606297517-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2025-01-11T14:38:20,433 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,37769,1736606297517-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2025-01-11T14:38:20,434 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(1385): Balancer post startup initialization complete, took 0 seconds 2025-01-11T14:38:20,434 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2025-01-11T14:38:20,435 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,37769,1736606297517-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:20,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7af16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-11T14:38:20,512 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2025-01-11T14:38:20,512 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2025-01-11T14:38:20,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request df3d9de72c82,37769,-1 for getting cluster id 2025-01-11T14:38:20,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2025-01-11T14:38:20,530 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '708b124d-3a3d-4cb0-a16c-f3e3b764637c' 2025-01-11T14:38:20,533 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2025-01-11T14:38:20,533 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "708b124d-3a3d-4cb0-a16c-f3e3b764637c" 2025-01-11T14:38:20,535 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a6be5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-11T14:38:20,535 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [df3d9de72c82,37769,-1] 2025-01-11T14:38:20,538 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2025-01-11T14:38:20,539 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:20,541 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2025-01-11T14:38:20,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49611c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-11T14:38:20,545 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2025-01-11T14:38:20,552 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=df3d9de72c82,38289,1736606298445, seqNum=-1] 2025-01-11T14:38:20,552 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2025-01-11T14:38:20,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2025-01-11T14:38:20,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=df3d9de72c82,37769,1736606297517 2025-01-11T14:38:20,578 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2025-01-11T14:38:20,585 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is df3d9de72c82,37769,1736606297517 2025-01-11T14:38:20,588 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@69e9e72 2025-01-11T14:38:20,589 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2025-01-11T14:38:20,591 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2025-01-11T14:38:20,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37769 {}] master.HMaster$4(2469): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1736606300575', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2025-01-11T14:38:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37769 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1736606300575 2025-01-11T14:38:20,605 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1736606300575 execute state=CREATE_TABLE_PRE_OPERATION 2025-01-11T14:38:20,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37769 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1736606300575" procId is: 4 2025-01-11T14:38:20,608 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:20,610 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1736606300575 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2025-01-11T14:38:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2025-01-11T14:38:20,632 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1736606300575-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2ce8f7c2094e486b4e00b3b9fa0e98b5, NAME => 'testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1736606300575', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a 2025-01-11T14:38:20,641 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1736606300575-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:20,641 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1736606300575-pool-0 {}] regionserver.HRegion(1722): Closing 2ce8f7c2094e486b4e00b3b9fa0e98b5, disabling compactions & flushes 2025-01-11T14:38:20,641 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1736606300575-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:20,641 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1736606300575-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:20,642 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1736606300575-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. after waiting 0 ms 2025-01-11T14:38:20,642 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1736606300575-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:20,642 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1736606300575-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:20,642 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1736606300575-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2ce8f7c2094e486b4e00b3b9fa0e98b5: Waiting for close lock at 1736606300641Disabling compacts and flushes for region at 1736606300641Disabling writes for close at 1736606300642 (+1 ms)Writing region close event to WAL at 1736606300642Closed at 1736606300642 2025-01-11T14:38:20,644 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1736606300575 execute state=CREATE_TABLE_ADD_TO_META 2025-01-11T14:38:20,649 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1736606300644"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1736606300644"}]},"ts":"1736606300644"} 2025-01-11T14:38:20,654 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2025-01-11T14:38:20,656 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1736606300575 execute state=CREATE_TABLE_ASSIGN_REGIONS 2025-01-11T14:38:20,659 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1736606300575","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1736606300656"}]},"ts":"1736606300656"} 2025-01-11T14:38:20,664 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1736606300575, state=ENABLING in hbase:meta 2025-01-11T14:38:20,664 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {df3d9de72c82=0} racks are {/default-rack=0} 2025-01-11T14:38:20,665 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2025-01-11T14:38:20,665 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2025-01-11T14:38:20,665 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2025-01-11T14:38:20,665 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2025-01-11T14:38:20,665 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2025-01-11T14:38:20,665 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2025-01-11T14:38:20,665 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2025-01-11T14:38:20,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1736606300575, region=2ce8f7c2094e486b4e00b3b9fa0e98b5, ASSIGN}] 2025-01-11T14:38:20,668 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1736606300575, region=2ce8f7c2094e486b4e00b3b9fa0e98b5, ASSIGN 2025-01-11T14:38:20,670 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1736606300575, region=2ce8f7c2094e486b4e00b3b9fa0e98b5, ASSIGN; state=OFFLINE, location=df3d9de72c82,33459,1736606298299; forceNewPlan=false, retain=false 2025-01-11T14:38:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2025-01-11T14:38:20,822 INFO [df3d9de72c82:37769 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2025-01-11T14:38:20,823 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2ce8f7c2094e486b4e00b3b9fa0e98b5, regionState=OPENING, regionLocation=df3d9de72c82,33459,1736606298299 2025-01-11T14:38:20,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1736606300575, region=2ce8f7c2094e486b4e00b3b9fa0e98b5, ASSIGN because future has completed 2025-01-11T14:38:20,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ce8f7c2094e486b4e00b3b9fa0e98b5, server=df3d9de72c82,33459,1736606298299}] 2025-01-11T14:38:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2025-01-11T14:38:20,982 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2025-01-11T14:38:20,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41749, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2025-01-11T14:38:20,988 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:20,988 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ce8f7c2094e486b4e00b3b9fa0e98b5, NAME => 'testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5.', STARTKEY => '', ENDKEY => ''} 2025-01-11T14:38:20,989 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1736606300575 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:20,989 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:20,990 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:20,990 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:20,993 INFO [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:20,999 INFO [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ce8f7c2094e486b4e00b3b9fa0e98b5 columnFamilyName family 2025-01-11T14:38:20,999 DEBUG [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:21,000 INFO [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] regionserver.HStore(327): Store=2ce8f7c2094e486b4e00b3b9fa0e98b5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2025-01-11T14:38:21,000 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:21,002 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:21,002 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:21,002 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:21,002 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:21,004 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:21,017 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2025-01-11T14:38:21,019 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2ce8f7c2094e486b4e00b3b9fa0e98b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68583838, jitterRate=0.021978825330734253}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2025-01-11T14:38:21,020 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:21,021 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2ce8f7c2094e486b4e00b3b9fa0e98b5: Running coprocessor pre-open hook at 1736606300990Writing region info on filesystem at 1736606300990Initializing all the Stores at 1736606300990Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606300991 (+1 ms)Cleaning up temporary data from old regions at 1736606301002 (+11 ms)Running coprocessor post-open hooks at 1736606301020 (+18 ms)Region opened successfully at 1736606301021 (+1 ms) 2025-01-11T14:38:21,023 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5., pid=6, masterSystemTime=1736606300981 2025-01-11T14:38:21,028 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:21,028 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:21,028 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2ce8f7c2094e486b4e00b3b9fa0e98b5, regionState=OPEN, openSeqNum=2, regionLocation=df3d9de72c82,33459,1736606298299 2025-01-11T14:38:21,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ce8f7c2094e486b4e00b3b9fa0e98b5, server=df3d9de72c82,33459,1736606298299 because future has completed 2025-01-11T14:38:21,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2025-01-11T14:38:21,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ce8f7c2094e486b4e00b3b9fa0e98b5, server=df3d9de72c82,33459,1736606298299 in 210 msec 2025-01-11T14:38:21,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2025-01-11T14:38:21,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1736606300575, region=2ce8f7c2094e486b4e00b3b9fa0e98b5, ASSIGN in 376 msec 2025-01-11T14:38:21,046 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1736606300575 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2025-01-11T14:38:21,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1736606300575","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1736606301046"}]},"ts":"1736606301046"} 2025-01-11T14:38:21,050 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1736606300575, state=ENABLED in hbase:meta 2025-01-11T14:38:21,051 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1736606300575 execute state=CREATE_TABLE_POST_OPERATION 2025-01-11T14:38:21,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1736606300575 in 452 msec 2025-01-11T14:38:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2025-01-11T14:38:21,251 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1736606300575 get assigned. Timeout = 60000ms 2025-01-11T14:38:21,251 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1736606300575 completed 2025-01-11T14:38:21,255 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2025-01-11T14:38:21,263 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1736606300575 assigned to meta. Checking AM states. 2025-01-11T14:38:21,264 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2025-01-11T14:38:21,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1736606300575 assigned. 2025-01-11T14:38:21,277 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1736606300575', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5., hostname=df3d9de72c82,33459,1736606298299, seqNum=2] 2025-01-11T14:38:21,278 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2025-01-11T14:38:21,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2025-01-11T14:38:21,297 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2ce8f7c2094e486b4e00b3b9fa0e98b5 1/1 column families, dataSize=78 B heapSize=480 B 2025-01-11T14:38:21,335 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e49889d263024a4fb5dfbcfb5648e424 2025-01-11T14:38:21,380 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/.tmp/family/e49889d263024a4fb5dfbcfb5648e424 is 43, key is row1/family:qf1/1/Put/seqid=0 2025-01-11T14:38:21,386 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/.tmp/family/e49889d263024a4fb5dfbcfb5648e424 2025-01-11T14:38:21,439 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/.tmp/family/e49889d263024a4fb5dfbcfb5648e424 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424 2025-01-11T14:38:21,441 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424, delay=972 ms 2025-01-11T14:38:21,444 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424, entries=2, sequenceid=6, filesize=4.9 K 2025-01-11T14:38:21,451 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 2ce8f7c2094e486b4e00b3b9fa0e98b5 in 155ms, sequenceid=6, compaction requested=false 2025-01-11T14:38:21,453 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1736606300575' 2025-01-11T14:38:21,455 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2ce8f7c2094e486b4e00b3b9fa0e98b5: 2025-01-11T14:38:22,416 DEBUG [hfile-prefetch-1736606301441 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key e49889d263024a4fb5dfbcfb5648e424_0. 2025-01-11T14:38:22,422 DEBUG [hfile-prefetch-1736606301441 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key e49889d263024a4fb5dfbcfb5648e424_117. 2025-01-11T14:38:22,423 DEBUG [hfile-prefetch-1736606301441 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424, with total blocks 2, and data blocks 1 2025-01-11T14:38:22,423 DEBUG [hfile-prefetch-1736606301441 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2025-01-11T14:38:22,424 DEBUG [hfile-prefetch-1736606301441 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2025-01-11T14:38:22,424 DEBUG [hfile-prefetch-1736606301441 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424. Will sleep 100ms and try the verification again. 2025-01-11T14:38:22,465 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-11T14:38:22,465 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-11T14:38:22,466 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2025-01-11T14:38:22,466 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-11T14:38:22,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37769 {}] master.HMaster(2425): Client=jenkins//172.17.0.2 move hri=2ce8f7c2094e486b4e00b3b9fa0e98b5, source=df3d9de72c82,33459,1736606298299, destination=df3d9de72c82,38289,1736606298445, warming up region on df3d9de72c82,38289,1736606298445 2025-01-11T14:38:22,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37769 {}] master.HMaster(2429): Client=jenkins//172.17.0.2 move hri=2ce8f7c2094e486b4e00b3b9fa0e98b5, source=df3d9de72c82,33459,1736606298299, destination=df3d9de72c82,38289,1736606298445, running balancer 2025-01-11T14:38:22,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38289 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38289 {}] regionserver.HRegion(7855): Warmup {ENCODED => 2ce8f7c2094e486b4e00b3b9fa0e98b5, NAME => 'testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5.', STARTKEY => '', ENDKEY => ''} 2025-01-11T14:38:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37769 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1736606300575, region=2ce8f7c2094e486b4e00b3b9fa0e98b5, REOPEN/MOVE 2025-01-11T14:38:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38289 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:22,476 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1736606300575, region=2ce8f7c2094e486b4e00b3b9fa0e98b5, REOPEN/MOVE 2025-01-11T14:38:22,477 INFO [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,478 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2ce8f7c2094e486b4e00b3b9fa0e98b5, regionState=CLOSING, regionLocation=df3d9de72c82,33459,1736606298299 2025-01-11T14:38:22,478 INFO [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ce8f7c2094e486b4e00b3b9fa0e98b5 columnFamilyName family 2025-01-11T14:38:22,478 DEBUG [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:22,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1736606300575, region=2ce8f7c2094e486b4e00b3b9fa0e98b5, REOPEN/MOVE because future has completed 2025-01-11T14:38:22,489 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2025-01-11T14:38:22,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2ce8f7c2094e486b4e00b3b9fa0e98b5, server=df3d9de72c82,33459,1736606298299}] 2025-01-11T14:38:22,496 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-11T14:38:22,496 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-11T14:38:22,497 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2025-01-11T14:38:22,497 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-11T14:38:22,497 DEBUG [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424 2025-01-11T14:38:22,497 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-11T14:38:22,497 INFO [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] regionserver.HStore(327): Store=2ce8f7c2094e486b4e00b3b9fa0e98b5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2025-01-11T14:38:22,497 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1736606302465 2025-01-11T14:38:22,498 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-11T14:38:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38289 {}] regionserver.HRegion(1722): Closing 2ce8f7c2094e486b4e00b3b9fa0e98b5, disabling compactions & flushes 2025-01-11T14:38:22,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38289 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38289 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38289 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. after waiting 0 ms 2025-01-11T14:38:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38289 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:22,501 DEBUG [StoreFileCloser-2ce8f7c2094e486b4e00b3b9fa0e98b5-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424 2025-01-11T14:38:22,506 WARN [hfile-prefetch-1736606301441 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2025-01-11T14:38:22,507 DEBUG [hfile-prefetch-1736606301441 {}] hfile.PrefetchExecutor(121): Prefetch completed for e49889d263024a4fb5dfbcfb5648e424 2025-01-11T14:38:22,508 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38289 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38289 {}] regionserver.HRegion(1676): Region close journal for 2ce8f7c2094e486b4e00b3b9fa0e98b5: Waiting for close lock at 1736606302498Disabling compacts and flushes for region at 1736606302498Disabling writes for close at 1736606302498Writing region close event to WAL at 1736606302508 (+10 ms)Closed at 1736606302508 2025-01-11T14:38:22,518 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-11T14:38:22,518 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1736606302496 2025-01-11T14:38:22,518 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-11T14:38:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37769 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2025-01-11T14:38:22,650 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,650 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2025-01-11T14:38:22,651 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 2ce8f7c2094e486b4e00b3b9fa0e98b5, disabling compactions & flushes 2025-01-11T14:38:22,651 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:22,651 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:22,651 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. after waiting 0 ms 2025-01-11T14:38:22,651 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:22,653 DEBUG [StoreFileCloser-2ce8f7c2094e486b4e00b3b9fa0e98b5-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file e49889d263024a4fb5dfbcfb5648e424, starting offset: 0, end offset: 9223372036854775807 2025-01-11T14:38:22,655 DEBUG [StoreFileCloser-2ce8f7c2094e486b4e00b3b9fa0e98b5-family-1 {}] bucket.BucketCache(856): removed key e49889d263024a4fb5dfbcfb5648e424_0 from back map with offset lock 2094080 in the evict process 2025-01-11T14:38:22,661 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2025-01-11T14:38:22,663 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:22,663 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 2ce8f7c2094e486b4e00b3b9fa0e98b5: Waiting for close lock at 1736606302651Running coprocessor pre-close hooks at 1736606302651Disabling compacts and flushes for region at 1736606302651Disabling writes for close at 1736606302651Writing region close event to WAL at 1736606302655 (+4 ms)Running coprocessor post-close hooks at 1736606302661 (+6 ms)Closed at 1736606302663 (+2 ms) 2025-01-11T14:38:22,663 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 2ce8f7c2094e486b4e00b3b9fa0e98b5 move to df3d9de72c82,38289,1736606298445 record at close sequenceid=6 2025-01-11T14:38:22,667 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,667 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2ce8f7c2094e486b4e00b3b9fa0e98b5, regionState=CLOSED 2025-01-11T14:38:22,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2ce8f7c2094e486b4e00b3b9fa0e98b5, server=df3d9de72c82,33459,1736606298299 because future has completed 2025-01-11T14:38:22,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2025-01-11T14:38:22,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 2ce8f7c2094e486b4e00b3b9fa0e98b5, server=df3d9de72c82,33459,1736606298299 in 182 msec 2025-01-11T14:38:22,674 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1736606300575, region=2ce8f7c2094e486b4e00b3b9fa0e98b5, REOPEN/MOVE; state=CLOSED, location=df3d9de72c82,38289,1736606298445; forceNewPlan=false, retain=false 2025-01-11T14:38:22,698 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-11T14:38:22,698 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-11T14:38:22,698 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2025-01-11T14:38:22,698 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-11T14:38:22,702 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-11T14:38:22,702 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1736606302698 2025-01-11T14:38:22,702 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-11T14:38:22,825 INFO [df3d9de72c82:37769 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2025-01-11T14:38:22,825 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2ce8f7c2094e486b4e00b3b9fa0e98b5, regionState=OPENING, regionLocation=df3d9de72c82,38289,1736606298445 2025-01-11T14:38:22,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1736606300575, region=2ce8f7c2094e486b4e00b3b9fa0e98b5, REOPEN/MOVE because future has completed 2025-01-11T14:38:22,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ce8f7c2094e486b4e00b3b9fa0e98b5, server=df3d9de72c82,38289,1736606298445}] 2025-01-11T14:38:22,985 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:22,985 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ce8f7c2094e486b4e00b3b9fa0e98b5, NAME => 'testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5.', STARTKEY => '', ENDKEY => ''} 2025-01-11T14:38:22,986 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1736606300575 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,986 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:22,986 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,986 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,987 INFO [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,988 INFO [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ce8f7c2094e486b4e00b3b9fa0e98b5 columnFamilyName family 2025-01-11T14:38:22,989 DEBUG [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:22,991 DEBUG [StoreFileOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424, delay=950 ms 2025-01-11T14:38:22,992 DEBUG [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424 2025-01-11T14:38:22,992 INFO [StoreOpener-2ce8f7c2094e486b4e00b3b9fa0e98b5-1 {}] regionserver.HStore(327): Store=2ce8f7c2094e486b4e00b3b9fa0e98b5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2025-01-11T14:38:22,993 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,993 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,994 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,994 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,994 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:22,999 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:23,000 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 2ce8f7c2094e486b4e00b3b9fa0e98b5; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65244995, jitterRate=-0.027773812413215637}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2025-01-11T14:38:23,000 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:23,001 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 2ce8f7c2094e486b4e00b3b9fa0e98b5: Running coprocessor pre-open hook at 1736606302986Writing region info on filesystem at 1736606302986Initializing all the Stores at 1736606302987 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606302987Cleaning up temporary data from old regions at 1736606302994 (+7 ms)Running coprocessor post-open hooks at 1736606303000 (+6 ms)Region opened successfully at 1736606303001 (+1 ms) 2025-01-11T14:38:23,002 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5., pid=9, masterSystemTime=1736606302982 2025-01-11T14:38:23,006 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:23,006 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:23,006 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2ce8f7c2094e486b4e00b3b9fa0e98b5, regionState=OPEN, openSeqNum=10, regionLocation=df3d9de72c82,38289,1736606298445 2025-01-11T14:38:23,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ce8f7c2094e486b4e00b3b9fa0e98b5, server=df3d9de72c82,38289,1736606298445 because future has completed 2025-01-11T14:38:23,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2025-01-11T14:38:23,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ce8f7c2094e486b4e00b3b9fa0e98b5, server=df3d9de72c82,38289,1736606298445 in 180 msec 2025-01-11T14:38:23,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1736606300575, region=2ce8f7c2094e486b4e00b3b9fa0e98b5, REOPEN/MOVE in 538 msec 2025-01-11T14:38:23,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2025-01-11T14:38:23,037 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2025-01-11T14:38:23,037 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2025-01-11T14:38:23,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:23,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:23,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2025-01-11T14:38:23,043 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2025-01-11T14:38:23,043 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=421165399, stopped=false 2025-01-11T14:38:23,043 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=df3d9de72c82,37769,1736606297517 2025-01-11T14:38:23,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:23,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:23,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:23,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:23,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:23,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:23,046 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2025-01-11T14:38:23,046 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:23,047 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2025-01-11T14:38:23,047 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3321) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3292) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:686) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2025-01-11T14:38:23,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:23,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:23,047 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'df3d9de72c82,33459,1736606298299' ***** 2025-01-11T14:38:23,047 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2025-01-11T14:38:23,047 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'df3d9de72c82,38289,1736606298445' ***** 2025-01-11T14:38:23,047 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2025-01-11T14:38:23,048 INFO [RS:0;df3d9de72c82:33459 {}] bucket.BucketCache(1738): Disabling cache 2025-01-11T14:38:23,048 INFO [RS:1;df3d9de72c82:38289 {}] bucket.BucketCache(1738): Disabling cache 2025-01-11T14:38:23,048 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-11T14:38:23,048 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-11T14:38:23,048 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-11T14:38:23,049 INFO [RS:1;df3d9de72c82:38289 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/bucket.persistence 2025-01-11T14:38:23,049 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-11T14:38:23,049 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-11T14:38:23,049 INFO [RS:0;df3d9de72c82:33459 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/bucket.persistence 2025-01-11T14:38:23,049 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:23,049 INFO [RS:0;df3d9de72c82:33459 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2025-01-11T14:38:23,049 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-11T14:38:23,049 INFO [RS:1;df3d9de72c82:38289 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2025-01-11T14:38:23,049 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2025-01-11T14:38:23,049 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-11T14:38:23,049 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-11T14:38:23,053 INFO [regionserver/df3d9de72c82:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2025-01-11T14:38:23,054 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2025-01-11T14:38:23,055 INFO [regionserver/df3d9de72c82:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2025-01-11T14:38:23,059 DEBUG [RS:0;df3d9de72c82:33459 {}] bucket.BucketCache(1487): Thread RS:0;df3d9de72c82:33459 started persisting bucket cache to file 2025-01-11T14:38:23,059 DEBUG [RS:0;df3d9de72c82:33459 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2025-01-11T14:38:23,059 DEBUG [RS:0;df3d9de72c82:33459 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-11T14:38:23,059 DEBUG [RS:1;df3d9de72c82:38289 {}] bucket.BucketCache(1487): Thread RS:1;df3d9de72c82:38289 started persisting bucket cache to file 2025-01-11T14:38:23,059 DEBUG [RS:1;df3d9de72c82:38289 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2025-01-11T14:38:23,059 DEBUG [RS:1;df3d9de72c82:38289 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-11T14:38:23,063 DEBUG [RS:0;df3d9de72c82:33459 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-11T14:38:23,063 DEBUG [RS:0;df3d9de72c82:33459 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1736606303059 2025-01-11T14:38:23,063 WARN [Thread-96 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedReader.ensureOpen(BufferedReader.java:123) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:321) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2025-01-11T14:38:23,063 DEBUG [RS:0;df3d9de72c82:33459 {}] bucket.BucketCache(1509): Thread RS:0;df3d9de72c82:33459 finished persisting bucket cache to file, renaming 2025-01-11T14:38:23,063 ERROR [RS:1;df3d9de72c82:38289 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2025-01-11T14:38:23,063 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.HeapMemoryManager(224): Stopping 2025-01-11T14:38:23,063 DEBUG [RS:1;df3d9de72c82:38289 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-11T14:38:23,064 DEBUG [RS:1;df3d9de72c82:38289 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1736606303059 2025-01-11T14:38:23,064 INFO [RS:0;df3d9de72c82:33459 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2025-01-11T14:38:23,064 DEBUG [RS:1;df3d9de72c82:38289 {}] bucket.BucketCache(1509): Thread RS:1;df3d9de72c82:38289 finished persisting bucket cache to file, renaming 2025-01-11T14:38:23,064 WARN [RS:1;df3d9de72c82:38289 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2025-01-11T14:38:23,064 INFO [RS:0;df3d9de72c82:33459 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2025-01-11T14:38:23,064 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer(959): stopping server df3d9de72c82,33459,1736606298299 2025-01-11T14:38:23,064 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HeapMemoryManager(224): Stopping 2025-01-11T14:38:23,064 INFO [RS:0;df3d9de72c82:33459 {}] hbase.HBaseServerBase(457): Close async cluster connection 2025-01-11T14:38:23,064 INFO [RS:0;df3d9de72c82:33459 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;df3d9de72c82:33459. 2025-01-11T14:38:23,064 INFO [RS:1;df3d9de72c82:38289 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2025-01-11T14:38:23,064 DEBUG [RS:0;df3d9de72c82:33459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:459) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2025-01-11T14:38:23,064 INFO [RS:1;df3d9de72c82:38289 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2025-01-11T14:38:23,064 DEBUG [RS:0;df3d9de72c82:33459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:23,064 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(3091): Received CLOSE for 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:23,064 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer(976): stopping server df3d9de72c82,33459,1736606298299; all regions closed. 2025-01-11T14:38:23,066 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(959): stopping server df3d9de72c82,38289,1736606298445 2025-01-11T14:38:23,066 INFO [RS:1;df3d9de72c82:38289 {}] hbase.HBaseServerBase(457): Close async cluster connection 2025-01-11T14:38:23,066 INFO [RS:1;df3d9de72c82:38289 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;df3d9de72c82:38289. 2025-01-11T14:38:23,066 DEBUG [RS:1;df3d9de72c82:38289 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:459) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2025-01-11T14:38:23,066 DEBUG [RS:1;df3d9de72c82:38289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:23,066 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2025-01-11T14:38:23,066 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2025-01-11T14:38:23,066 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2025-01-11T14:38:23,066 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2025-01-11T14:38:23,067 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2025-01-11T14:38:23,067 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2ce8f7c2094e486b4e00b3b9fa0e98b5, disabling compactions & flushes 2025-01-11T14:38:23,067 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:23,067 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:23,067 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. after waiting 0 ms 2025-01-11T14:38:23,067 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:23,067 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2025-01-11T14:38:23,067 DEBUG [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(1325): Online Regions={2ce8f7c2094e486b4e00b3b9fa0e98b5=testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5., 1588230740=hbase:meta,,1.1588230740} 2025-01-11T14:38:23,068 DEBUG [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2ce8f7c2094e486b4e00b3b9fa0e98b5 2025-01-11T14:38:23,068 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2025-01-11T14:38:23,068 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2025-01-11T14:38:23,068 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2025-01-11T14:38:23,068 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2025-01-11T14:38:23,068 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2025-01-11T14:38:23,068 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2025-01-11T14:38:23,069 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2025-01-11T14:38:23,070 DEBUG [RS:0;df3d9de72c82:33459 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs 2025-01-11T14:38:23,070 INFO [RS:0;df3d9de72c82:33459 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL df3d9de72c82%2C33459%2C1736606298299:(num 1736606299810) 2025-01-11T14:38:23,070 DEBUG [RS:0;df3d9de72c82:33459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:23,070 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.LeaseManager(133): Closed leases 2025-01-11T14:38:23,070 INFO [RS:0;df3d9de72c82:33459 {}] hbase.HBaseServerBase(440): Shutdown chores and chore service 2025-01-11T14:38:23,070 INFO [RS:0;df3d9de72c82:33459 {}] hbase.ChoreService(370): Chore service for: regionserver/df3d9de72c82:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2025-01-11T14:38:23,071 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2025-01-11T14:38:23,071 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2025-01-11T14:38:23,071 INFO [regionserver/df3d9de72c82:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2025-01-11T14:38:23,071 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2025-01-11T14:38:23,071 INFO [RS:0;df3d9de72c82:33459 {}] hbase.HBaseServerBase(450): Shutdown executor service 2025-01-11T14:38:23,072 INFO [RS:0;df3d9de72c82:33459 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33459 2025-01-11T14:38:23,073 DEBUG [StoreFileCloser-2ce8f7c2094e486b4e00b3b9fa0e98b5-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/family/e49889d263024a4fb5dfbcfb5648e424 2025-01-11T14:38:23,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/df3d9de72c82,33459,1736606298299 2025-01-11T14:38:23,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2025-01-11T14:38:23,077 INFO [RS:0;df3d9de72c82:33459 {}] hbase.HBaseServerBase(481): Close zookeeper 2025-01-11T14:38:23,079 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [df3d9de72c82,33459,1736606298299] 2025-01-11T14:38:23,081 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnRegionMove1736606300575/2ce8f7c2094e486b4e00b3b9fa0e98b5/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2025-01-11T14:38:23,082 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:23,082 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2ce8f7c2094e486b4e00b3b9fa0e98b5: Waiting for close lock at 1736606303066Running coprocessor pre-close hooks at 1736606303067 (+1 ms)Disabling compacts and flushes for region at 1736606303067Disabling writes for close at 1736606303067Writing region close event to WAL at 1736606303075 (+8 ms)Running coprocessor post-close hooks at 1736606303082 (+7 ms)Closed at 1736606303082 2025-01-11T14:38:23,082 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5. 2025-01-11T14:38:23,083 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/df3d9de72c82,33459,1736606298299 already deleted, retry=false 2025-01-11T14:38:23,083 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; df3d9de72c82,33459,1736606298299 expired; onlineServers=1 2025-01-11T14:38:23,086 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 68396fb769fc4b1fa72eb4d08c116bb4 2025-01-11T14:38:23,111 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/info/68396fb769fc4b1fa72eb4d08c116bb4 is 205, key is testBlockEvictionOnRegionMove1736606300575,,1736606300592.2ce8f7c2094e486b4e00b3b9fa0e98b5./info:regioninfo/1736606303006/Put/seqid=0 2025-01-11T14:38:23,112 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/info/68396fb769fc4b1fa72eb4d08c116bb4 2025-01-11T14:38:23,129 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7b4cd51891cd4971b9a259238f29d7b9 2025-01-11T14:38:23,144 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/ns/7b4cd51891cd4971b9a259238f29d7b9 is 43, key is default/ns:d/1736606300340/Put/seqid=0 2025-01-11T14:38:23,145 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/ns/7b4cd51891cd4971b9a259238f29d7b9 2025-01-11T14:38:23,157 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 94188171ca7e42f19f8c9e29139485a4 2025-01-11T14:38:23,174 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/table/94188171ca7e42f19f8c9e29139485a4 is 78, key is testBlockEvictionOnRegionMove1736606300575/table:state/1736606301046/Put/seqid=0 2025-01-11T14:38:23,175 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/table/94188171ca7e42f19f8c9e29139485a4 2025-01-11T14:38:23,178 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/info/68396fb769fc4b1fa72eb4d08c116bb4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/info/68396fb769fc4b1fa72eb4d08c116bb4 2025-01-11T14:38:23,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:23,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x101696b37120001, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:23,180 INFO [RS:0;df3d9de72c82:33459 {}] hbase.HBaseServerBase(488): Close table descriptors 2025-01-11T14:38:23,180 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/info/68396fb769fc4b1fa72eb4d08c116bb4, delay=975 ms 2025-01-11T14:38:23,180 INFO [RS:0;df3d9de72c82:33459 {}] regionserver.HRegionServer(1031): Exiting; stopping=df3d9de72c82,33459,1736606298299; zookeeper connection closed. 2025-01-11T14:38:23,181 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7911f914 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7911f914 2025-01-11T14:38:23,182 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/info/68396fb769fc4b1fa72eb4d08c116bb4, entries=15, sequenceid=15, filesize=7.6 K 2025-01-11T14:38:23,182 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/ns/7b4cd51891cd4971b9a259238f29d7b9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/ns/7b4cd51891cd4971b9a259238f29d7b9 2025-01-11T14:38:23,183 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/ns/7b4cd51891cd4971b9a259238f29d7b9, delay=966 ms 2025-01-11T14:38:23,185 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/ns/7b4cd51891cd4971b9a259238f29d7b9, entries=2, sequenceid=15, filesize=5.0 K 2025-01-11T14:38:23,186 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/table/94188171ca7e42f19f8c9e29139485a4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/table/94188171ca7e42f19f8c9e29139485a4 2025-01-11T14:38:23,187 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/table/94188171ca7e42f19f8c9e29139485a4, delay=986 ms 2025-01-11T14:38:23,189 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/table/94188171ca7e42f19f8c9e29139485a4, entries=2, sequenceid=15, filesize=5.3 K 2025-01-11T14:38:23,191 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 122ms, sequenceid=15, compaction requested=false 2025-01-11T14:38:23,191 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2025-01-11T14:38:23,193 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/info/68396fb769fc4b1fa72eb4d08c116bb4 2025-01-11T14:38:23,194 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/ns/7b4cd51891cd4971b9a259238f29d7b9 2025-01-11T14:38:23,195 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/table/94188171ca7e42f19f8c9e29139485a4 2025-01-11T14:38:23,201 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2025-01-11T14:38:23,202 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2025-01-11T14:38:23,202 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2025-01-11T14:38:23,202 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1736606303068Running coprocessor pre-close hooks at 1736606303068Disabling compacts and flushes for region at 1736606303068Disabling writes for close at 1736606303068Obtaining lock to block concurrent updates at 1736606303069 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1736606303069Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1736606303069Flushing stores of hbase:meta,,1.1588230740 at 1736606303069Flushing 1588230740/info: creating writer at 1736606303070 (+1 ms)Flushing 1588230740/info: appending metadata at 1736606303110 (+40 ms)Flushing 1588230740/info: closing flushed file at 1736606303110Flushing 1588230740/ns: creating writer at 1736606303117 (+7 ms)Flushing 1588230740/ns: appending metadata at 1736606303143 (+26 ms)Flushing 1588230740/ns: closing flushed file at 1736606303143Flushing 1588230740/table: creating writer at 1736606303148 (+5 ms)Flushing 1588230740/table: appending metadata at 1736606303173 (+25 ms)Flushing 1588230740/table: closing flushed file at 1736606303173Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a7c7fc5: reopening flushed file at 1736606303178 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d16670c: reopening flushed file at 1736606303182 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c852419: reopening flushed file at 1736606303186 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 122ms, sequenceid=15, compaction requested=false at 1736606303191 (+5 ms)Writing region close event to WAL at 1736606303195 (+4 ms)Running coprocessor post-close hooks at 1736606303201 (+6 ms)Closed at 1736606303202 (+1 ms) 2025-01-11T14:38:23,203 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2025-01-11T14:38:23,268 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(976): stopping server df3d9de72c82,38289,1736606298445; all regions closed. 2025-01-11T14:38:23,270 DEBUG [RS:1;df3d9de72c82:38289 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs 2025-01-11T14:38:23,271 INFO [RS:1;df3d9de72c82:38289 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL df3d9de72c82%2C38289%2C1736606298445.meta:.meta(num 1736606300210) 2025-01-11T14:38:23,273 DEBUG [RS:1;df3d9de72c82:38289 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs 2025-01-11T14:38:23,273 INFO [RS:1;df3d9de72c82:38289 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL df3d9de72c82%2C38289%2C1736606298445:(num 1736606299812) 2025-01-11T14:38:23,273 DEBUG [RS:1;df3d9de72c82:38289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:23,273 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.LeaseManager(133): Closed leases 2025-01-11T14:38:23,273 INFO [RS:1;df3d9de72c82:38289 {}] hbase.HBaseServerBase(440): Shutdown chores and chore service 2025-01-11T14:38:23,273 INFO [RS:1;df3d9de72c82:38289 {}] hbase.ChoreService(370): Chore service for: regionserver/df3d9de72c82:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2025-01-11T14:38:23,273 INFO [RS:1;df3d9de72c82:38289 {}] hbase.HBaseServerBase(450): Shutdown executor service 2025-01-11T14:38:23,273 INFO [regionserver/df3d9de72c82:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2025-01-11T14:38:23,273 INFO [RS:1;df3d9de72c82:38289 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38289 2025-01-11T14:38:23,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2025-01-11T14:38:23,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/df3d9de72c82,38289,1736606298445 2025-01-11T14:38:23,276 INFO [RS:1;df3d9de72c82:38289 {}] hbase.HBaseServerBase(481): Close zookeeper 2025-01-11T14:38:23,277 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [df3d9de72c82,38289,1736606298445] 2025-01-11T14:38:23,279 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/df3d9de72c82,38289,1736606298445 already deleted, retry=false 2025-01-11T14:38:23,279 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; df3d9de72c82,38289,1736606298445 expired; onlineServers=0 2025-01-11T14:38:23,279 INFO [RegionServerTracker-0 {}] master.HMaster(3336): ***** STOPPING master 'df3d9de72c82,37769,1736606297517' ***** 2025-01-11T14:38:23,279 INFO [RegionServerTracker-0 {}] master.HMaster(3338): STOPPED: Cluster shutdown set; onlineServer=0 2025-01-11T14:38:23,279 INFO [M:0;df3d9de72c82:37769 {}] hbase.HBaseServerBase(457): Close async cluster connection 2025-01-11T14:38:23,279 INFO [M:0;df3d9de72c82:37769 {}] hbase.HBaseServerBase(440): Shutdown chores and chore service 2025-01-11T14:38:23,279 DEBUG [M:0;df3d9de72c82:37769 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2025-01-11T14:38:23,279 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2025-01-11T14:38:23,280 DEBUG [M:0;df3d9de72c82:37769 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2025-01-11T14:38:23,280 DEBUG [master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.large.0-1736606299456 {}] cleaner.HFileCleaner(306): Exit Thread[master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.large.0-1736606299456,5,FailOnTimeoutGroup] 2025-01-11T14:38:23,280 DEBUG [master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.small.0-1736606299458 {}] cleaner.HFileCleaner(306): Exit Thread[master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.small.0-1736606299458,5,FailOnTimeoutGroup] 2025-01-11T14:38:23,280 INFO [M:0;df3d9de72c82:37769 {}] hbase.ChoreService(370): Chore service for: master/df3d9de72c82:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2025-01-11T14:38:23,280 INFO [M:0;df3d9de72c82:37769 {}] hbase.HBaseServerBase(450): Shutdown executor service 2025-01-11T14:38:23,280 DEBUG [M:0;df3d9de72c82:37769 {}] master.HMaster(1806): Stopping service threads 2025-01-11T14:38:23,280 INFO [M:0;df3d9de72c82:37769 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2025-01-11T14:38:23,280 INFO [M:0;df3d9de72c82:37769 {}] procedure2.ProcedureExecutor(723): Stopping 2025-01-11T14:38:23,281 INFO [M:0;df3d9de72c82:37769 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2025-01-11T14:38:23,281 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2025-01-11T14:38:23,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2025-01-11T14:38:23,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:23,282 DEBUG [M:0;df3d9de72c82:37769 {}] zookeeper.ZKUtil(347): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2025-01-11T14:38:23,282 WARN [M:0;df3d9de72c82:37769 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2025-01-11T14:38:23,282 INFO [M:0;df3d9de72c82:37769 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/.lastflushedseqids 2025-01-11T14:38:23,296 INFO [M:0;df3d9de72c82:37769 {}] assignment.AssignmentManager(395): Stopping assignment manager 2025-01-11T14:38:23,296 INFO [M:0;df3d9de72c82:37769 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2025-01-11T14:38:23,296 DEBUG [M:0;df3d9de72c82:37769 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2025-01-11T14:38:23,296 INFO [M:0;df3d9de72c82:37769 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:23,296 DEBUG [M:0;df3d9de72c82:37769 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:23,296 DEBUG [M:0;df3d9de72c82:37769 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2025-01-11T14:38:23,296 DEBUG [M:0;df3d9de72c82:37769 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:23,297 INFO [M:0;df3d9de72c82:37769 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2025-01-11T14:38:23,309 DEBUG [M:0;df3d9de72c82:37769 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d82d6c5b7b974d6bb965adc161d69545 2025-01-11T14:38:23,324 DEBUG [M:0;df3d9de72c82:37769 {}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d82d6c5b7b974d6bb965adc161d69545 is 82, key is hbase:meta,,1/info:regioninfo/1736606300276/Put/seqid=0 2025-01-11T14:38:23,325 INFO [M:0;df3d9de72c82:37769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d82d6c5b7b974d6bb965adc161d69545 2025-01-11T14:38:23,335 DEBUG [M:0;df3d9de72c82:37769 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b65808a437334315a4bfc7387f3a1560 2025-01-11T14:38:23,353 DEBUG [M:0;df3d9de72c82:37769 {}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b65808a437334315a4bfc7387f3a1560 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1736606301053/Put/seqid=0 2025-01-11T14:38:23,353 INFO [M:0;df3d9de72c82:37769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b65808a437334315a4bfc7387f3a1560 2025-01-11T14:38:23,362 DEBUG [M:0;df3d9de72c82:37769 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 355ad1749a7d42488b5fe95252e42cd6 2025-01-11T14:38:23,376 DEBUG [M:0;df3d9de72c82:37769 {}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/355ad1749a7d42488b5fe95252e42cd6 is 69, key is df3d9de72c82,33459,1736606298299/rs:state/1736606299546/Put/seqid=0 2025-01-11T14:38:23,376 INFO [M:0;df3d9de72c82:37769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/355ad1749a7d42488b5fe95252e42cd6 2025-01-11T14:38:23,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:23,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38289-0x101696b37120002, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:23,378 INFO [RS:1;df3d9de72c82:38289 {}] hbase.HBaseServerBase(488): Close table descriptors 2025-01-11T14:38:23,378 INFO [RS:1;df3d9de72c82:38289 {}] regionserver.HRegionServer(1031): Exiting; stopping=df3d9de72c82,38289,1736606298445; zookeeper connection closed. 2025-01-11T14:38:23,378 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@957c073 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@957c073 2025-01-11T14:38:23,379 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2025-01-11T14:38:23,379 DEBUG [M:0;df3d9de72c82:37769 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d82d6c5b7b974d6bb965adc161d69545 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d82d6c5b7b974d6bb965adc161d69545 2025-01-11T14:38:23,381 INFO [M:0;df3d9de72c82:37769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d82d6c5b7b974d6bb965adc161d69545, entries=8, sequenceid=84, filesize=5.5 K 2025-01-11T14:38:23,381 DEBUG [M:0;df3d9de72c82:37769 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b65808a437334315a4bfc7387f3a1560 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b65808a437334315a4bfc7387f3a1560 2025-01-11T14:38:23,383 INFO [M:0;df3d9de72c82:37769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b65808a437334315a4bfc7387f3a1560, entries=9, sequenceid=84, filesize=6.5 K 2025-01-11T14:38:23,384 DEBUG [M:0;df3d9de72c82:37769 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/355ad1749a7d42488b5fe95252e42cd6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/355ad1749a7d42488b5fe95252e42cd6 2025-01-11T14:38:23,385 INFO [M:0;df3d9de72c82:37769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/355ad1749a7d42488b5fe95252e42cd6, entries=2, sequenceid=84, filesize=5.1 K 2025-01-11T14:38:23,386 INFO [M:0;df3d9de72c82:37769 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false 2025-01-11T14:38:23,392 INFO [M:0;df3d9de72c82:37769 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:23,392 DEBUG [M:0;df3d9de72c82:37769 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1736606303296Disabling compacts and flushes for region at 1736606303296Disabling writes for close at 1736606303296Obtaining lock to block concurrent updates at 1736606303297 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1736606303297Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1736606303297Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1736606303297Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1736606303298 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1736606303323 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1736606303323Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1736606303327 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1736606303352 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1736606303352Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1736606303355 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1736606303375 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1736606303375Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b34bbe8: reopening flushed file at 1736606303379 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f033657: reopening flushed file at 1736606303381 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32e19fc1: reopening flushed file at 1736606303383 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false at 1736606303386 (+3 ms)Writing region close event to WAL at 1736606303392 (+6 ms)Closed at 1736606303392 2025-01-11T14:38:23,394 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2025-01-11T14:38:23,394 INFO [M:0;df3d9de72c82:37769 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2025-01-11T14:38:23,394 INFO [M:0;df3d9de72c82:37769 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37769 2025-01-11T14:38:23,395 INFO [M:0;df3d9de72c82:37769 {}] hbase.HBaseServerBase(481): Close zookeeper 2025-01-11T14:38:23,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:23,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37769-0x101696b37120000, quorum=127.0.0.1:58962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:23,498 INFO [M:0;df3d9de72c82:37769 {}] hbase.HBaseServerBase(488): Close table descriptors 2025-01-11T14:38:23,505 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2025-01-11T14:38:23,531 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2025-01-11T14:38:23,531 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:23,542 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: master/df3d9de72c82:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1736606303180 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/df3d9de72c82:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1736606302991 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1736606303183 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/df3d9de72c82:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1736606301441 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/df3d9de72c82:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=246 (was 189) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5197 (was 4368) - AvailableMemoryMB LEAK? - 2025-01-11T14:38:23,549 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=246, ProcessCount=11, AvailableMemoryMB=5195 2025-01-11T14:38:23,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/hadoop-log-dir so I do NOT create it in target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40 2025-01-11T14:38:23,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/hadoop-tmp-dir so I do NOT create it in target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40 2025-01-11T14:38:23,549 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:23,553 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/cluster_d0fdeb1c-23d7-0474-1d4c-dede039e4ef8, deleteOnExit=true 2025-01-11T14:38:23,557 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/cluster_d0fdeb1c-23d7-0474-1d4c-dede039e4ef8/zookeeper_0, clientPort=64262, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/cluster_d0fdeb1c-23d7-0474-1d4c-dede039e4ef8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/cluster_d0fdeb1c-23d7-0474-1d4c-dede039e4ef8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2025-01-11T14:38:23,559 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64262 2025-01-11T14:38:23,573 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a with version=8 2025-01-11T14:38:23,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/hbase-staging 2025-01-11T14:38:23,577 INFO [Time-limited test {}] client.ConnectionUtils(128): master/df3d9de72c82:0 server-side Connection retries=45 2025-01-11T14:38:23,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:23,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:23,577 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2025-01-11T14:38:23,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:23,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2025-01-11T14:38:23,577 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2025-01-11T14:38:23,578 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-11T14:38:23,578 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43135 2025-01-11T14:38:23,580 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43135 connecting to ZooKeeper ensemble=127.0.0.1:64262 2025-01-11T14:38:23,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:431350x0, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2025-01-11T14:38:23,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43135-0x101696b4fe20000 connected 2025-01-11T14:38:23,609 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:23,609 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:23,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:23,611 INFO [Time-limited test {}] master.HMaster(536): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a, hbase.cluster.distributed=false 2025-01-11T14:38:23,614 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2025-01-11T14:38:23,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43135 2025-01-11T14:38:23,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43135 2025-01-11T14:38:23,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43135 2025-01-11T14:38:23,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43135 2025-01-11T14:38:23,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43135 2025-01-11T14:38:23,644 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/df3d9de72c82:0 server-side Connection retries=45 2025-01-11T14:38:23,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:23,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:23,645 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2025-01-11T14:38:23,645 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:23,645 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2025-01-11T14:38:23,645 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2025-01-11T14:38:23,645 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-11T14:38:23,646 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44219 2025-01-11T14:38:23,648 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44219 connecting to ZooKeeper ensemble=127.0.0.1:64262 2025-01-11T14:38:23,649 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:23,649 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:23,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442190x0, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2025-01-11T14:38:23,654 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44219-0x101696b4fe20001 connected 2025-01-11T14:38:23,654 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:23,654 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2025-01-11T14:38:23,655 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.cache 2025-01-11T14:38:23,655 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2025-01-11T14:38:23,656 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2025-01-11T14:38:23,656 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2025-01-11T14:38:23,656 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2025-01-11T14:38:23,656 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-11T14:38:23,656 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-11T14:38:23,657 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2025-01-11T14:38:23,657 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-11T14:38:23,657 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.persistence is complete. 2025-01-11T14:38:23,658 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2025-01-11T14:38:23,660 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2025-01-11T14:38:23,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44219 2025-01-11T14:38:23,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44219 2025-01-11T14:38:23,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44219 2025-01-11T14:38:23,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44219 2025-01-11T14:38:23,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44219 2025-01-11T14:38:23,692 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/df3d9de72c82:0 server-side Connection retries=45 2025-01-11T14:38:23,692 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:23,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:23,693 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2025-01-11T14:38:23,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:23,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2025-01-11T14:38:23,693 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2025-01-11T14:38:23,693 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-11T14:38:23,694 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41723 2025-01-11T14:38:23,696 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41723 connecting to ZooKeeper ensemble=127.0.0.1:64262 2025-01-11T14:38:23,696 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:23,697 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:23,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417230x0, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2025-01-11T14:38:23,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:417230x0, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:23,703 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2025-01-11T14:38:23,705 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41723-0x101696b4fe20002 connected 2025-01-11T14:38:23,708 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.cache 2025-01-11T14:38:23,708 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2025-01-11T14:38:23,708 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2025-01-11T14:38:23,708 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2025-01-11T14:38:23,709 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2025-01-11T14:38:23,709 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-11T14:38:23,710 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-11T14:38:23,710 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.persistence is complete. 2025-01-11T14:38:23,710 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-11T14:38:23,712 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2025-01-11T14:38:23,714 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2025-01-11T14:38:23,716 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2025-01-11T14:38:23,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41723 2025-01-11T14:38:23,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41723 2025-01-11T14:38:23,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41723 2025-01-11T14:38:23,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41723 2025-01-11T14:38:23,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41723 2025-01-11T14:38:23,756 DEBUG [M:0;df3d9de72c82:43135 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;df3d9de72c82:43135 2025-01-11T14:38:23,756 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(2525): Adding backup master ZNode /hbase/backup-masters/df3d9de72c82,43135,1736606303576 2025-01-11T14:38:23,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:23,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:23,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:23,763 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/df3d9de72c82,43135,1736606303576 2025-01-11T14:38:23,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2025-01-11T14:38:23,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:23,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2025-01-11T14:38:23,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:23,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:23,768 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2025-01-11T14:38:23,769 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/df3d9de72c82,43135,1736606303576 from backup master directory 2025-01-11T14:38:23,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/df3d9de72c82,43135,1736606303576 2025-01-11T14:38:23,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:23,771 WARN [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2025-01-11T14:38:23,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:23,771 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=df3d9de72c82,43135,1736606303576 2025-01-11T14:38:23,773 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/hbase.id] with ID: 0fc479bb-18e4-4d64-aec5-40e5b85cc15a 2025-01-11T14:38:23,773 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/.tmp/hbase.id 2025-01-11T14:38:23,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2025-01-11T14:38:23,785 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/hbase.id] 2025-01-11T14:38:23,849 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:23,849 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2025-01-11T14:38:23,851 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2025-01-11T14:38:23,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:23,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:23,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:23,866 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2025-01-11T14:38:23,868 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2025-01-11T14:38:23,869 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2025-01-11T14:38:23,912 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store 2025-01-11T14:38:23,922 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:23,922 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2025-01-11T14:38:23,922 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:23,922 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:23,922 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2025-01-11T14:38:23,922 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:23,922 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:23,922 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1736606303922Disabling compacts and flushes for region at 1736606303922Disabling writes for close at 1736606303922Writing region close event to WAL at 1736606303922Closed at 1736606303922 2025-01-11T14:38:23,923 WARN [master/df3d9de72c82:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/.initializing 2025-01-11T14:38:23,923 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/WALs/df3d9de72c82,43135,1736606303576 2025-01-11T14:38:23,936 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2025-01-11T14:38:23,941 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=df3d9de72c82%2C43135%2C1736606303576, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/WALs/df3d9de72c82,43135,1736606303576, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/oldWALs, maxLogs=10 2025-01-11T14:38:23,972 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/WALs/df3d9de72c82,43135,1736606303576/df3d9de72c82%2C43135%2C1736606303576.1736606303941 2025-01-11T14:38:23,972 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2025-01-11T14:38:23,972 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2025-01-11T14:38:23,973 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:23,973 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:23,973 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:23,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:23,980 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2025-01-11T14:38:23,981 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:23,981 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:23,981 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:23,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2025-01-11T14:38:23,986 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:23,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2025-01-11T14:38:23,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:23,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2025-01-11T14:38:23,993 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:23,994 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2025-01-11T14:38:23,994 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:23,998 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2025-01-11T14:38:23,998 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:23,998 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2025-01-11T14:38:23,999 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:23,999 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:23,999 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:24,000 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:24,000 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:24,000 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2025-01-11T14:38:24,001 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2025-01-11T14:38:24,016 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2025-01-11T14:38:24,016 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73778435, jitterRate=0.0993843525648117}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2025-01-11T14:38:24,018 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1736606303973Initializing all the Stores at 1736606303974 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606303974Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606303974Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606303974Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606303974Cleaning up temporary data from old regions at 1736606304000 (+26 ms)Region opened successfully at 1736606304017 (+17 ms) 2025-01-11T14:38:24,020 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2025-01-11T14:38:24,030 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5907d9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=df3d9de72c82/172.17.0.2:0 2025-01-11T14:38:24,032 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(923): No meta location available on zookeeper, skip migrating... 2025-01-11T14:38:24,032 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2025-01-11T14:38:24,032 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2025-01-11T14:38:24,032 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2025-01-11T14:38:24,033 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2025-01-11T14:38:24,033 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2025-01-11T14:38:24,033 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2025-01-11T14:38:24,035 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2025-01-11T14:38:24,036 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2025-01-11T14:38:24,038 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2025-01-11T14:38:24,038 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2025-01-11T14:38:24,039 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2025-01-11T14:38:24,041 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2025-01-11T14:38:24,041 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2025-01-11T14:38:24,042 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2025-01-11T14:38:24,044 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2025-01-11T14:38:24,045 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2025-01-11T14:38:24,046 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2025-01-11T14:38:24,049 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2025-01-11T14:38:24,051 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2025-01-11T14:38:24,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:24,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:24,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:24,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,055 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(867): Active/primary master=df3d9de72c82,43135,1736606303576, sessionid=0x101696b4fe20000, setting cluster-up flag (Was=false) 2025-01-11T14:38:24,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,067 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2025-01-11T14:38:24,069 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=df3d9de72c82,43135,1736606303576 2025-01-11T14:38:24,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,081 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2025-01-11T14:38:24,083 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=df3d9de72c82,43135,1736606303576 2025-01-11T14:38:24,084 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2025-01-11T14:38:24,087 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2025-01-11T14:38:24,087 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2025-01-11T14:38:24,087 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2025-01-11T14:38:24,087 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: df3d9de72c82,43135,1736606303576 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2025-01-11T14:38:24,089 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/df3d9de72c82:0, corePoolSize=5, maxPoolSize=5 2025-01-11T14:38:24,089 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/df3d9de72c82:0, corePoolSize=5, maxPoolSize=5 2025-01-11T14:38:24,089 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/df3d9de72c82:0, corePoolSize=5, maxPoolSize=5 2025-01-11T14:38:24,090 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/df3d9de72c82:0, corePoolSize=5, maxPoolSize=5 2025-01-11T14:38:24,090 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/df3d9de72c82:0, corePoolSize=10, maxPoolSize=10 2025-01-11T14:38:24,090 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,090 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/df3d9de72c82:0, corePoolSize=2, maxPoolSize=2 2025-01-11T14:38:24,090 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,098 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2025-01-11T14:38:24,098 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2025-01-11T14:38:24,098 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:24,099 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2025-01-11T14:38:24,103 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1736606334103 2025-01-11T14:38:24,103 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2025-01-11T14:38:24,104 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2025-01-11T14:38:24,104 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2025-01-11T14:38:24,104 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2025-01-11T14:38:24,104 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2025-01-11T14:38:24,104 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2025-01-11T14:38:24,104 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,116 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2025-01-11T14:38:24,116 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2025-01-11T14:38:24,116 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2025-01-11T14:38:24,120 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2025-01-11T14:38:24,120 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2025-01-11T14:38:24,120 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.large.0-1736606304120,5,FailOnTimeoutGroup] 2025-01-11T14:38:24,121 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.small.0-1736606304120,5,FailOnTimeoutGroup] 2025-01-11T14:38:24,121 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,122 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(1752): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2025-01-11T14:38:24,122 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,122 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,137 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2025-01-11T14:38:24,137 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a 2025-01-11T14:38:24,145 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(746): ClusterId : 0fc479bb-18e4-4d64-aec5-40e5b85cc15a 2025-01-11T14:38:24,145 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2025-01-11T14:38:24,147 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(746): ClusterId : 0fc479bb-18e4-4d64-aec5-40e5b85cc15a 2025-01-11T14:38:24,147 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2025-01-11T14:38:24,151 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2025-01-11T14:38:24,151 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2025-01-11T14:38:24,151 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2025-01-11T14:38:24,151 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2025-01-11T14:38:24,154 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2025-01-11T14:38:24,155 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2025-01-11T14:38:24,155 DEBUG [RS:0;df3d9de72c82:44219 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@baeb49f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=df3d9de72c82/172.17.0.2:0 2025-01-11T14:38:24,155 DEBUG [RS:1;df3d9de72c82:41723 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d31d7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=df3d9de72c82/172.17.0.2:0 2025-01-11T14:38:24,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:24,170 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2025-01-11T14:38:24,181 DEBUG [RS:0;df3d9de72c82:44219 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;df3d9de72c82:44219 2025-01-11T14:38:24,181 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2025-01-11T14:38:24,181 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2025-01-11T14:38:24,182 DEBUG [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(832): About to register with Master. 2025-01-11T14:38:24,184 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(2659): reportForDuty to master=df3d9de72c82,43135,1736606303576 with port=44219, startcode=1736606303643 2025-01-11T14:38:24,184 DEBUG [RS:0;df3d9de72c82:44219 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2025-01-11T14:38:24,189 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2025-01-11T14:38:24,190 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:24,190 DEBUG [RS:1;df3d9de72c82:41723 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;df3d9de72c82:41723 2025-01-11T14:38:24,190 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2025-01-11T14:38:24,190 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2025-01-11T14:38:24,190 DEBUG [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(832): About to register with Master. 2025-01-11T14:38:24,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:24,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2025-01-11T14:38:24,191 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(2659): reportForDuty to master=df3d9de72c82,43135,1736606303576 with port=41723, startcode=1736606303692 2025-01-11T14:38:24,191 DEBUG [RS:1;df3d9de72c82:41723 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2025-01-11T14:38:24,197 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40411, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2025-01-11T14:38:24,197 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2025-01-11T14:38:24,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:24,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:24,198 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43135 {}] master.ServerManager(363): Checking decommissioned status of RegionServer df3d9de72c82,44219,1736606303643 2025-01-11T14:38:24,198 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43135 {}] master.ServerManager(517): Registering regionserver=df3d9de72c82,44219,1736606303643 2025-01-11T14:38:24,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2025-01-11T14:38:24,199 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49041, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2025-01-11T14:38:24,200 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43135 {}] master.ServerManager(363): Checking decommissioned status of RegionServer df3d9de72c82,41723,1736606303692 2025-01-11T14:38:24,200 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43135 {}] master.ServerManager(517): Registering regionserver=df3d9de72c82,41723,1736606303692 2025-01-11T14:38:24,200 DEBUG [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a 2025-01-11T14:38:24,200 DEBUG [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2025-01-11T14:38:24,200 DEBUG [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2025-01-11T14:38:24,202 DEBUG [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a 2025-01-11T14:38:24,202 DEBUG [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2025-01-11T14:38:24,202 DEBUG [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2025-01-11T14:38:24,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2025-01-11T14:38:24,208 DEBUG [RS:0;df3d9de72c82:44219 {}] zookeeper.ZKUtil(111): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/df3d9de72c82,44219,1736606303643 2025-01-11T14:38:24,208 WARN [RS:0;df3d9de72c82:44219 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2025-01-11T14:38:24,208 INFO [RS:0;df3d9de72c82:44219 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2025-01-11T14:38:24,208 DEBUG [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,44219,1736606303643 2025-01-11T14:38:24,208 DEBUG [RS:1;df3d9de72c82:41723 {}] zookeeper.ZKUtil(111): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/df3d9de72c82,41723,1736606303692 2025-01-11T14:38:24,208 WARN [RS:1;df3d9de72c82:41723 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2025-01-11T14:38:24,208 INFO [RS:1;df3d9de72c82:41723 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2025-01-11T14:38:24,209 DEBUG [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,41723,1736606303692 2025-01-11T14:38:24,217 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [df3d9de72c82,44219,1736606303643] 2025-01-11T14:38:24,217 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [df3d9de72c82,41723,1736606303692] 2025-01-11T14:38:24,224 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2025-01-11T14:38:24,225 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2025-01-11T14:38:24,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:24,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:24,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2025-01-11T14:38:24,237 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2025-01-11T14:38:24,238 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2025-01-11T14:38:24,238 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2025-01-11T14:38:24,238 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:24,239 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:24,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2025-01-11T14:38:24,239 INFO [RS:1;df3d9de72c82:41723 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2025-01-11T14:38:24,239 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740 2025-01-11T14:38:24,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740 2025-01-11T14:38:24,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2025-01-11T14:38:24,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2025-01-11T14:38:24,241 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2025-01-11T14:38:24,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2025-01-11T14:38:24,248 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2025-01-11T14:38:24,249 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2025-01-11T14:38:24,250 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,250 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,250 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,250 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,250 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,250 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,250 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/df3d9de72c82:0, corePoolSize=2, maxPoolSize=2 2025-01-11T14:38:24,251 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,251 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,251 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,251 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,251 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,251 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,251 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/df3d9de72c82:0, corePoolSize=3, maxPoolSize=3 2025-01-11T14:38:24,251 DEBUG [RS:1;df3d9de72c82:41723 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/df3d9de72c82:0, corePoolSize=3, maxPoolSize=3 2025-01-11T14:38:24,256 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2025-01-11T14:38:24,256 INFO [RS:0;df3d9de72c82:44219 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2025-01-11T14:38:24,257 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,257 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,257 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,257 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,257 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,257 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,257 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,41723,1736606303692-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2025-01-11T14:38:24,258 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2025-01-11T14:38:24,259 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2025-01-11T14:38:24,260 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,260 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,260 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,260 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,260 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,261 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,261 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/df3d9de72c82:0, corePoolSize=2, maxPoolSize=2 2025-01-11T14:38:24,261 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,261 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,262 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,262 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,262 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,262 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:24,262 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/df3d9de72c82:0, corePoolSize=3, maxPoolSize=3 2025-01-11T14:38:24,263 DEBUG [RS:0;df3d9de72c82:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/df3d9de72c82:0, corePoolSize=3, maxPoolSize=3 2025-01-11T14:38:24,269 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2025-01-11T14:38:24,269 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63316103, jitterRate=-0.05651654303073883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2025-01-11T14:38:24,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1736606304167Initializing all the Stores at 1736606304168 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606304168Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606304170 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606304170Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606304170Cleaning up temporary data from old regions at 1736606304240 (+70 ms)Region opened successfully at 1736606304270 (+30 ms) 2025-01-11T14:38:24,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2025-01-11T14:38:24,271 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2025-01-11T14:38:24,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2025-01-11T14:38:24,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2025-01-11T14:38:24,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2025-01-11T14:38:24,273 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2025-01-11T14:38:24,273 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1736606304271Disabling compacts and flushes for region at 1736606304271Disabling writes for close at 1736606304271Writing region close event to WAL at 1736606304273 (+2 ms)Closed at 1736606304273 2025-01-11T14:38:24,274 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2025-01-11T14:38:24,275 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2025-01-11T14:38:24,276 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2025-01-11T14:38:24,277 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,41723,1736606303692-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,277 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,277 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.Replication(171): df3d9de72c82,41723,1736606303692 started 2025-01-11T14:38:24,280 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2025-01-11T14:38:24,280 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,280 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,280 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,280 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,280 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,44219,1736606303643-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2025-01-11T14:38:24,282 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2025-01-11T14:38:24,283 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2025-01-11T14:38:24,298 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,299 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(1482): Serving as df3d9de72c82,41723,1736606303692, RpcServer on df3d9de72c82/172.17.0.2:41723, sessionid=0x101696b4fe20002 2025-01-11T14:38:24,299 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2025-01-11T14:38:24,299 DEBUG [RS:1;df3d9de72c82:41723 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager df3d9de72c82,41723,1736606303692 2025-01-11T14:38:24,299 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'df3d9de72c82,41723,1736606303692' 2025-01-11T14:38:24,299 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2025-01-11T14:38:24,300 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2025-01-11T14:38:24,301 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2025-01-11T14:38:24,301 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2025-01-11T14:38:24,301 DEBUG [RS:1;df3d9de72c82:41723 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager df3d9de72c82,41723,1736606303692 2025-01-11T14:38:24,301 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'df3d9de72c82,41723,1736606303692' 2025-01-11T14:38:24,301 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2025-01-11T14:38:24,303 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2025-01-11T14:38:24,303 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2025-01-11T14:38:24,303 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,44219,1736606303643-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,304 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,304 DEBUG [RS:1;df3d9de72c82:41723 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2025-01-11T14:38:24,304 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.Replication(171): df3d9de72c82,44219,1736606303643 started 2025-01-11T14:38:24,304 INFO [RS:1;df3d9de72c82:41723 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2025-01-11T14:38:24,304 INFO [RS:1;df3d9de72c82:41723 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2025-01-11T14:38:24,319 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,319 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(1482): Serving as df3d9de72c82,44219,1736606303643, RpcServer on df3d9de72c82/172.17.0.2:44219, sessionid=0x101696b4fe20001 2025-01-11T14:38:24,319 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2025-01-11T14:38:24,319 DEBUG [RS:0;df3d9de72c82:44219 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager df3d9de72c82,44219,1736606303643 2025-01-11T14:38:24,319 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'df3d9de72c82,44219,1736606303643' 2025-01-11T14:38:24,319 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2025-01-11T14:38:24,320 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2025-01-11T14:38:24,321 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2025-01-11T14:38:24,321 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2025-01-11T14:38:24,321 DEBUG [RS:0;df3d9de72c82:44219 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager df3d9de72c82,44219,1736606303643 2025-01-11T14:38:24,321 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'df3d9de72c82,44219,1736606303643' 2025-01-11T14:38:24,321 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2025-01-11T14:38:24,322 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2025-01-11T14:38:24,322 DEBUG [RS:0;df3d9de72c82:44219 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2025-01-11T14:38:24,322 INFO [RS:0;df3d9de72c82:44219 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2025-01-11T14:38:24,322 INFO [RS:0;df3d9de72c82:44219 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2025-01-11T14:38:24,405 INFO [RS:1;df3d9de72c82:41723 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2025-01-11T14:38:24,405 INFO [RS:1;df3d9de72c82:41723 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=df3d9de72c82%2C41723%2C1736606303692, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,41723,1736606303692, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs, maxLogs=55 2025-01-11T14:38:24,423 INFO [RS:0;df3d9de72c82:44219 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2025-01-11T14:38:24,424 INFO [RS:0;df3d9de72c82:44219 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=df3d9de72c82%2C44219%2C1736606303643, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,44219,1736606303643, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs, maxLogs=55 2025-01-11T14:38:24,428 INFO [RS:1;df3d9de72c82:41723 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,41723,1736606303692/df3d9de72c82%2C41723%2C1736606303692.1736606304407 2025-01-11T14:38:24,428 DEBUG [RS:1;df3d9de72c82:41723 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2025-01-11T14:38:24,433 DEBUG [df3d9de72c82:43135 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2025-01-11T14:38:24,434 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=df3d9de72c82,41723,1736606303692 2025-01-11T14:38:24,436 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as df3d9de72c82,41723,1736606303692, state=OPENING 2025-01-11T14:38:24,437 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2025-01-11T14:38:24,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:24,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:24,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:24,441 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:24,442 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2025-01-11T14:38:24,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=df3d9de72c82,41723,1736606303692}] 2025-01-11T14:38:24,451 INFO [RS:0;df3d9de72c82:44219 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,44219,1736606303643/df3d9de72c82%2C44219%2C1736606303643.1736606304425 2025-01-11T14:38:24,456 DEBUG [RS:0;df3d9de72c82:44219 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2025-01-11T14:38:24,596 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2025-01-11T14:38:24,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41035, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2025-01-11T14:38:24,603 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2025-01-11T14:38:24,603 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2025-01-11T14:38:24,604 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2025-01-11T14:38:24,605 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=df3d9de72c82%2C41723%2C1736606303692.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,41723,1736606303692, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs, maxLogs=55 2025-01-11T14:38:24,632 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,41723,1736606303692/df3d9de72c82%2C41723%2C1736606303692.meta.1736606304606.meta 2025-01-11T14:38:24,632 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2025-01-11T14:38:24,633 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2025-01-11T14:38:24,633 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2025-01-11T14:38:24,633 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2025-01-11T14:38:24,634 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2025-01-11T14:38:24,634 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2025-01-11T14:38:24,634 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:24,634 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2025-01-11T14:38:24,634 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2025-01-11T14:38:24,636 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2025-01-11T14:38:24,637 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2025-01-11T14:38:24,637 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:24,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:24,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2025-01-11T14:38:24,638 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2025-01-11T14:38:24,639 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:24,639 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:24,639 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2025-01-11T14:38:24,640 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2025-01-11T14:38:24,640 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:24,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:24,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2025-01-11T14:38:24,641 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2025-01-11T14:38:24,641 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:24,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2025-01-11T14:38:24,641 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2025-01-11T14:38:24,642 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740 2025-01-11T14:38:24,643 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740 2025-01-11T14:38:24,643 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2025-01-11T14:38:24,643 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2025-01-11T14:38:24,644 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2025-01-11T14:38:24,645 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2025-01-11T14:38:24,646 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63688574, jitterRate=-0.0509662926197052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2025-01-11T14:38:24,646 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2025-01-11T14:38:24,647 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1736606304635Writing region info on filesystem at 1736606304635Initializing all the Stores at 1736606304635Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606304635Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606304636 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606304636Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1736606304636Cleaning up temporary data from old regions at 1736606304643 (+7 ms)Running coprocessor post-open hooks at 1736606304646 (+3 ms)Region opened successfully at 1736606304647 (+1 ms) 2025-01-11T14:38:24,648 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1736606304595 2025-01-11T14:38:24,652 DEBUG [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2025-01-11T14:38:24,652 INFO [RS_OPEN_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2025-01-11T14:38:24,652 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=df3d9de72c82,41723,1736606303692 2025-01-11T14:38:24,653 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as df3d9de72c82,41723,1736606303692, state=OPEN 2025-01-11T14:38:24,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2025-01-11T14:38:24,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2025-01-11T14:38:24,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2025-01-11T14:38:24,656 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:24,656 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=df3d9de72c82,41723,1736606303692 2025-01-11T14:38:24,656 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:24,656 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2025-01-11T14:38:24,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2025-01-11T14:38:24,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=df3d9de72c82,41723,1736606303692 in 214 msec 2025-01-11T14:38:24,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2025-01-11T14:38:24,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 379 msec 2025-01-11T14:38:24,663 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2025-01-11T14:38:24,663 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2025-01-11T14:38:24,668 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2025-01-11T14:38:24,668 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=df3d9de72c82,41723,1736606303692, seqNum=-1] 2025-01-11T14:38:24,668 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2025-01-11T14:38:24,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49369, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2025-01-11T14:38:24,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 589 msec 2025-01-11T14:38:24,676 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(1134): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1736606304676, completionTime=-1 2025-01-11T14:38:24,676 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2025-01-11T14:38:24,676 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2025-01-11T14:38:24,679 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2025-01-11T14:38:24,679 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1736606364679 2025-01-11T14:38:24,679 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1736606424679 2025-01-11T14:38:24,679 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2025-01-11T14:38:24,679 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,43135,1736606303576-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,679 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,43135,1736606303576-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,679 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,43135,1736606303576-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,680 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-df3d9de72c82:43135, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,680 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,680 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,682 DEBUG [master/df3d9de72c82:0.Chore.1 {}] janitor.CatalogJanitor(180): 2025-01-11T14:38:24,685 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(1250): Master has completed initialization 0.914sec 2025-01-11T14:38:24,686 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2025-01-11T14:38:24,686 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2025-01-11T14:38:24,686 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2025-01-11T14:38:24,686 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2025-01-11T14:38:24,686 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2025-01-11T14:38:24,686 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,43135,1736606303576-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2025-01-11T14:38:24,686 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,43135,1736606303576-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2025-01-11T14:38:24,687 DEBUG [master/df3d9de72c82:0:becomeActiveMaster {}] master.HMaster(1385): Balancer post startup initialization complete, took 0 seconds 2025-01-11T14:38:24,687 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2025-01-11T14:38:24,687 INFO [master/df3d9de72c82:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,43135,1736606303576-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:24,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@619da01d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-11T14:38:24,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request df3d9de72c82,43135,-1 for getting cluster id 2025-01-11T14:38:24,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2025-01-11T14:38:24,749 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0fc479bb-18e4-4d64-aec5-40e5b85cc15a' 2025-01-11T14:38:24,750 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2025-01-11T14:38:24,750 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0fc479bb-18e4-4d64-aec5-40e5b85cc15a" 2025-01-11T14:38:24,751 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fa87e7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-11T14:38:24,751 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [df3d9de72c82,43135,-1] 2025-01-11T14:38:24,751 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2025-01-11T14:38:24,752 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:24,754 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2025-01-11T14:38:24,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b5e2fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2025-01-11T14:38:24,756 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2025-01-11T14:38:24,758 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=df3d9de72c82,41723,1736606303692, seqNum=-1] 2025-01-11T14:38:24,758 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2025-01-11T14:38:24,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2025-01-11T14:38:24,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=df3d9de72c82,43135,1736606303576 2025-01-11T14:38:24,763 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2025-01-11T14:38:24,765 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is df3d9de72c82,43135,1736606303576 2025-01-11T14:38:24,765 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@babde79 2025-01-11T14:38:24,765 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2025-01-11T14:38:24,775 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2025-01-11T14:38:24,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43135 {}] master.HMaster$4(2469): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1736606304763', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2025-01-11T14:38:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43135 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1736606304763 2025-01-11T14:38:24,779 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1736606304763 execute state=CREATE_TABLE_PRE_OPERATION 2025-01-11T14:38:24,779 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:24,780 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1736606304763 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2025-01-11T14:38:24,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43135 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1736606304763" procId is: 4 2025-01-11T14:38:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43135 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2025-01-11T14:38:24,802 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1736606304763-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f1c63d9675757a36716ce777981a4e99, NAME => 'testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1736606304763', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a 2025-01-11T14:38:24,814 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1736606304763-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:24,814 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1736606304763-pool-0 {}] regionserver.HRegion(1722): Closing f1c63d9675757a36716ce777981a4e99, disabling compactions & flushes 2025-01-11T14:38:24,814 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1736606304763-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:24,814 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1736606304763-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:24,814 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1736606304763-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. after waiting 0 ms 2025-01-11T14:38:24,814 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1736606304763-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:24,814 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1736606304763-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:24,814 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1736606304763-pool-0 {}] regionserver.HRegion(1676): Region close journal for f1c63d9675757a36716ce777981a4e99: Waiting for close lock at 1736606304814Disabling compacts and flushes for region at 1736606304814Disabling writes for close at 1736606304814Writing region close event to WAL at 1736606304814Closed at 1736606304814 2025-01-11T14:38:24,816 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1736606304763 execute state=CREATE_TABLE_ADD_TO_META 2025-01-11T14:38:24,816 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1736606304816"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1736606304816"}]},"ts":"1736606304816"} 2025-01-11T14:38:24,819 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2025-01-11T14:38:24,820 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1736606304763 execute state=CREATE_TABLE_ASSIGN_REGIONS 2025-01-11T14:38:24,821 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1736606304763","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1736606304820"}]},"ts":"1736606304820"} 2025-01-11T14:38:24,824 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1736606304763, state=ENABLING in hbase:meta 2025-01-11T14:38:24,824 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {df3d9de72c82=0} racks are {/default-rack=0} 2025-01-11T14:38:24,825 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2025-01-11T14:38:24,825 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2025-01-11T14:38:24,825 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2025-01-11T14:38:24,825 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2025-01-11T14:38:24,825 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2025-01-11T14:38:24,825 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2025-01-11T14:38:24,825 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2025-01-11T14:38:24,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1736606304763, region=f1c63d9675757a36716ce777981a4e99, ASSIGN}] 2025-01-11T14:38:24,826 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1736606304763, region=f1c63d9675757a36716ce777981a4e99, ASSIGN 2025-01-11T14:38:24,827 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1736606304763, region=f1c63d9675757a36716ce777981a4e99, ASSIGN; state=OFFLINE, location=df3d9de72c82,44219,1736606303643; forceNewPlan=false, retain=false 2025-01-11T14:38:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43135 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2025-01-11T14:38:24,978 INFO [df3d9de72c82:43135 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2025-01-11T14:38:24,978 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f1c63d9675757a36716ce777981a4e99, regionState=OPENING, regionLocation=df3d9de72c82,44219,1736606303643 2025-01-11T14:38:24,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1736606304763, region=f1c63d9675757a36716ce777981a4e99, ASSIGN because future has completed 2025-01-11T14:38:24,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1c63d9675757a36716ce777981a4e99, server=df3d9de72c82,44219,1736606303643}] 2025-01-11T14:38:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43135 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2025-01-11T14:38:25,136 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2025-01-11T14:38:25,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50895, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2025-01-11T14:38:25,141 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:25,142 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f1c63d9675757a36716ce777981a4e99, NAME => 'testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99.', STARTKEY => '', ENDKEY => ''} 2025-01-11T14:38:25,142 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1736606304763 f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:25,142 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:25,143 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:25,143 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:25,144 INFO [StoreOpener-f1c63d9675757a36716ce777981a4e99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:25,150 INFO [StoreOpener-f1c63d9675757a36716ce777981a4e99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1c63d9675757a36716ce777981a4e99 columnFamilyName family 2025-01-11T14:38:25,150 DEBUG [StoreOpener-f1c63d9675757a36716ce777981a4e99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:25,151 INFO [StoreOpener-f1c63d9675757a36716ce777981a4e99-1 {}] regionserver.HStore(327): Store=f1c63d9675757a36716ce777981a4e99/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2025-01-11T14:38:25,151 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:25,152 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:25,152 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:25,152 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:25,152 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:25,153 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:25,163 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2025-01-11T14:38:25,163 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f1c63d9675757a36716ce777981a4e99; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59606440, jitterRate=-0.11179482936859131}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2025-01-11T14:38:25,163 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:25,164 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f1c63d9675757a36716ce777981a4e99: Running coprocessor pre-open hook at 1736606305143Writing region info on filesystem at 1736606305143Initializing all the Stores at 1736606305143Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606305143Cleaning up temporary data from old regions at 1736606305152 (+9 ms)Running coprocessor post-open hooks at 1736606305163 (+11 ms)Region opened successfully at 1736606305164 (+1 ms) 2025-01-11T14:38:25,165 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99., pid=6, masterSystemTime=1736606305135 2025-01-11T14:38:25,168 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:25,168 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:25,169 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f1c63d9675757a36716ce777981a4e99, regionState=OPEN, openSeqNum=2, regionLocation=df3d9de72c82,44219,1736606303643 2025-01-11T14:38:25,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1c63d9675757a36716ce777981a4e99, server=df3d9de72c82,44219,1736606303643 because future has completed 2025-01-11T14:38:25,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2025-01-11T14:38:25,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f1c63d9675757a36716ce777981a4e99, server=df3d9de72c82,44219,1736606303643 in 191 msec 2025-01-11T14:38:25,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2025-01-11T14:38:25,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1736606304763, region=f1c63d9675757a36716ce777981a4e99, ASSIGN in 350 msec 2025-01-11T14:38:25,179 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1736606304763 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2025-01-11T14:38:25,179 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1736606304763","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1736606305179"}]},"ts":"1736606305179"} 2025-01-11T14:38:25,182 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1736606304763, state=ENABLED in hbase:meta 2025-01-11T14:38:25,183 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1736606304763 execute state=CREATE_TABLE_POST_OPERATION 2025-01-11T14:38:25,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1736606304763 in 407 msec 2025-01-11T14:38:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43135 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2025-01-11T14:38:25,409 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1736606304763 completed 2025-01-11T14:38:25,409 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1736606304763 get assigned. Timeout = 60000ms 2025-01-11T14:38:25,410 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2025-01-11T14:38:25,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1736606304763 assigned to meta. Checking AM states. 2025-01-11T14:38:25,415 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2025-01-11T14:38:25,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1736606304763 assigned. 2025-01-11T14:38:25,420 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1736606304763', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99., hostname=df3d9de72c82,44219,1736606303643, seqNum=2] 2025-01-11T14:38:25,420 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2025-01-11T14:38:25,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2025-01-11T14:38:25,428 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f1c63d9675757a36716ce777981a4e99 1/1 column families, dataSize=78 B heapSize=480 B 2025-01-11T14:38:25,441 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 98122ed62bc84df1811498ce18b48ad9 2025-01-11T14:38:25,456 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/.tmp/family/98122ed62bc84df1811498ce18b48ad9 is 43, key is row1/family:qf1/1/Put/seqid=0 2025-01-11T14:38:25,457 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/.tmp/family/98122ed62bc84df1811498ce18b48ad9 2025-01-11T14:38:25,461 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/.tmp/family/98122ed62bc84df1811498ce18b48ad9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/family/98122ed62bc84df1811498ce18b48ad9 2025-01-11T14:38:25,462 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/family/98122ed62bc84df1811498ce18b48ad9, delay=998 ms 2025-01-11T14:38:25,464 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/family/98122ed62bc84df1811498ce18b48ad9, entries=2, sequenceid=6, filesize=4.9 K 2025-01-11T14:38:25,465 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for f1c63d9675757a36716ce777981a4e99 in 37ms, sequenceid=6, compaction requested=false 2025-01-11T14:38:25,465 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1736606304763' 2025-01-11T14:38:25,466 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f1c63d9675757a36716ce777981a4e99: 2025-01-11T14:38:25,747 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2025-01-11T14:38:26,462 DEBUG [hfile-prefetch-1736606302991 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 98122ed62bc84df1811498ce18b48ad9_0. 2025-01-11T14:38:26,463 DEBUG [hfile-prefetch-1736606302991 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 98122ed62bc84df1811498ce18b48ad9_117. 2025-01-11T14:38:26,463 DEBUG [hfile-prefetch-1736606302991 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/family/98122ed62bc84df1811498ce18b48ad9, with total blocks 2, and data blocks 1 2025-01-11T14:38:26,463 DEBUG [hfile-prefetch-1736606302991 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2025-01-11T14:38:26,463 DEBUG [hfile-prefetch-1736606302991 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/family/98122ed62bc84df1811498ce18b48ad9, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2025-01-11T14:38:26,463 DEBUG [hfile-prefetch-1736606302991 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/family/98122ed62bc84df1811498ce18b48ad9. Will sleep 100ms and try the verification again. 2025-01-11T14:38:26,466 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(514): Stopping Thread[RS:0;df3d9de72c82:44219,5,FailOnTimeoutGroup] 2025-01-11T14:38:26,466 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'df3d9de72c82,44219,1736606303643' ***** 2025-01-11T14:38:26,466 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2025-01-11T14:38:26,467 INFO [RS:0;df3d9de72c82:44219 {}] bucket.BucketCache(1738): Disabling cache 2025-01-11T14:38:26,467 INFO [RS:0;df3d9de72c82:44219 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.persistence 2025-01-11T14:38:26,467 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-11T14:38:26,468 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-11T14:38:26,471 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-11T14:38:26,471 INFO [RS:0;df3d9de72c82:44219 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2025-01-11T14:38:26,471 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-11T14:38:26,479 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2025-01-11T14:38:26,481 DEBUG [RS:0;df3d9de72c82:44219 {}] bucket.BucketCache(1487): Thread RS:0;df3d9de72c82:44219 started persisting bucket cache to file 2025-01-11T14:38:26,481 DEBUG [RS:0;df3d9de72c82:44219 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2025-01-11T14:38:26,481 DEBUG [RS:0;df3d9de72c82:44219 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-11T14:38:26,485 DEBUG [RS:0;df3d9de72c82:44219 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-11T14:38:26,485 DEBUG [RS:0;df3d9de72c82:44219 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1736606306481 2025-01-11T14:38:26,485 DEBUG [RS:0;df3d9de72c82:44219 {}] bucket.BucketCache(1509): Thread RS:0;df3d9de72c82:44219 finished persisting bucket cache to file, renaming 2025-01-11T14:38:26,485 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.HeapMemoryManager(224): Stopping 2025-01-11T14:38:26,486 INFO [RS:0;df3d9de72c82:44219 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2025-01-11T14:38:26,486 INFO [RS:0;df3d9de72c82:44219 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2025-01-11T14:38:26,486 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(3091): Received CLOSE for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:26,486 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(959): stopping server df3d9de72c82,44219,1736606303643 2025-01-11T14:38:26,486 INFO [regionserver/df3d9de72c82:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2025-01-11T14:38:26,486 INFO [RS:0;df3d9de72c82:44219 {}] hbase.HBaseServerBase(457): Close async cluster connection 2025-01-11T14:38:26,486 INFO [RS:0;df3d9de72c82:44219 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;df3d9de72c82:44219. 2025-01-11T14:38:26,486 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2025-01-11T14:38:26,486 DEBUG [RS:0;df3d9de72c82:44219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:459) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2025-01-11T14:38:26,487 DEBUG [RS:0;df3d9de72c82:44219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:26,487 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f1c63d9675757a36716ce777981a4e99, disabling compactions & flushes 2025-01-11T14:38:26,487 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:26,487 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2025-01-11T14:38:26,487 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:26,487 DEBUG [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(1325): Online Regions={f1c63d9675757a36716ce777981a4e99=testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99.} 2025-01-11T14:38:26,487 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. after waiting 0 ms 2025-01-11T14:38:26,487 DEBUG [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(1351): Waiting on f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:26,487 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:26,488 DEBUG [StoreFileCloser-f1c63d9675757a36716ce777981a4e99-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/family/98122ed62bc84df1811498ce18b48ad9 2025-01-11T14:38:26,488 WARN [hfile-prefetch-1736606302991 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/family/98122ed62bc84df1811498ce18b48ad9, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2025-01-11T14:38:26,488 DEBUG [hfile-prefetch-1736606302991 {}] hfile.PrefetchExecutor(121): Prefetch completed for 98122ed62bc84df1811498ce18b48ad9 2025-01-11T14:38:26,493 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2025-01-11T14:38:26,494 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:26,494 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f1c63d9675757a36716ce777981a4e99: Waiting for close lock at 1736606306486Running coprocessor pre-close hooks at 1736606306486Disabling compacts and flushes for region at 1736606306487 (+1 ms)Disabling writes for close at 1736606306487Writing region close event to WAL at 1736606306488 (+1 ms)Running coprocessor post-close hooks at 1736606306494 (+6 ms)Closed at 1736606306494 2025-01-11T14:38:26,494 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:26,511 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-11T14:38:26,511 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-11T14:38:26,511 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2025-01-11T14:38:26,511 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-11T14:38:26,515 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-11T14:38:26,515 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1736606306511 2025-01-11T14:38:26,516 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-11T14:38:26,687 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(976): stopping server df3d9de72c82,44219,1736606303643; all regions closed. 2025-01-11T14:38:26,689 DEBUG [RS:0;df3d9de72c82:44219 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs 2025-01-11T14:38:26,689 INFO [RS:0;df3d9de72c82:44219 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL df3d9de72c82%2C44219%2C1736606303643:(num 1736606304425) 2025-01-11T14:38:26,689 DEBUG [RS:0;df3d9de72c82:44219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:26,689 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.LeaseManager(133): Closed leases 2025-01-11T14:38:26,690 INFO [RS:0;df3d9de72c82:44219 {}] hbase.HBaseServerBase(440): Shutdown chores and chore service 2025-01-11T14:38:26,690 INFO [RS:0;df3d9de72c82:44219 {}] hbase.ChoreService(370): Chore service for: regionserver/df3d9de72c82:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2025-01-11T14:38:26,690 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2025-01-11T14:38:26,690 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2025-01-11T14:38:26,690 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2025-01-11T14:38:26,690 INFO [RS:0;df3d9de72c82:44219 {}] hbase.HBaseServerBase(450): Shutdown executor service 2025-01-11T14:38:26,690 INFO [regionserver/df3d9de72c82:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2025-01-11T14:38:26,690 INFO [RS:0;df3d9de72c82:44219 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44219 2025-01-11T14:38:26,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2025-01-11T14:38:26,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/df3d9de72c82,44219,1736606303643 2025-01-11T14:38:26,698 INFO [RS:0;df3d9de72c82:44219 {}] hbase.HBaseServerBase(481): Close zookeeper 2025-01-11T14:38:26,698 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [df3d9de72c82,44219,1736606303643] 2025-01-11T14:38:26,700 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/df3d9de72c82,44219,1736606303643 already deleted, retry=false 2025-01-11T14:38:26,700 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of df3d9de72c82,44219,1736606303643 on df3d9de72c82,43135,1736606303576 2025-01-11T14:38:26,705 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure df3d9de72c82,44219,1736606303643, splitWal=true, meta=false 2025-01-11T14:38:26,708 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for df3d9de72c82,44219,1736606303643 (carryingMeta=false) df3d9de72c82,44219,1736606303643/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7b8696c3[Write locks = 1, Read locks = 0], oldState=ONLINE. 2025-01-11T14:38:26,708 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure df3d9de72c82,44219,1736606303643, splitWal=true, meta=false 2025-01-11T14:38:26,710 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): df3d9de72c82,44219,1736606303643 had 1 regions 2025-01-11T14:38:26,712 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure df3d9de72c82,44219,1736606303643, splitWal=true, meta=false, isMeta: false 2025-01-11T14:38:26,713 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,44219,1736606303643-splitting 2025-01-11T14:38:26,714 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,44219,1736606303643-splitting dir is empty, no logs to split. 2025-01-11T14:38:26,714 INFO [PEWorker-2 {}] master.SplitWALManager(105): df3d9de72c82,44219,1736606303643 WAL count=0, meta=false 2025-01-11T14:38:26,717 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,44219,1736606303643-splitting dir is empty, no logs to split. 2025-01-11T14:38:26,717 INFO [PEWorker-2 {}] master.SplitWALManager(105): df3d9de72c82,44219,1736606303643 WAL count=0, meta=false 2025-01-11T14:38:26,717 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if df3d9de72c82,44219,1736606303643 WAL splitting is done? wals=0, meta=false 2025-01-11T14:38:26,718 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for df3d9de72c82,44219,1736606303643 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,44219,1736606303643-splitting does not exist 2025-01-11T14:38:26,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1736606304763, region=f1c63d9675757a36716ce777981a4e99, ASSIGN}] 2025-01-11T14:38:26,721 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1736606304763, region=f1c63d9675757a36716ce777981a4e99, ASSIGN 2025-01-11T14:38:26,722 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1736606304763, region=f1c63d9675757a36716ce777981a4e99, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2025-01-11T14:38:26,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:26,800 INFO [RS:0;df3d9de72c82:44219 {}] hbase.HBaseServerBase(488): Close table descriptors 2025-01-11T14:38:26,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x101696b4fe20001, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:26,800 INFO [RS:0;df3d9de72c82:44219 {}] regionserver.HRegionServer(1031): Exiting; stopping=df3d9de72c82,44219,1736606303643; zookeeper connection closed. 2025-01-11T14:38:26,800 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5cc84546 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5cc84546 2025-01-11T14:38:26,873 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=f1c63d9675757a36716ce777981a4e99, regionState=OPENING, regionLocation=df3d9de72c82,41723,1736606303692 2025-01-11T14:38:26,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1736606304763, region=f1c63d9675757a36716ce777981a4e99, ASSIGN because future has completed 2025-01-11T14:38:26,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1c63d9675757a36716ce777981a4e99, server=df3d9de72c82,41723,1736606303692}] 2025-01-11T14:38:26,983 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/df3d9de72c82:0 server-side Connection retries=45 2025-01-11T14:38:26,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:26,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:26,983 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2025-01-11T14:38:26,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2025-01-11T14:38:26,983 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2025-01-11T14:38:26,984 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2025-01-11T14:38:26,984 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2025-01-11T14:38:26,984 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46115 2025-01-11T14:38:26,986 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46115 connecting to ZooKeeper ensemble=127.0.0.1:64262 2025-01-11T14:38:26,986 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:26,986 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:26,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461150x0, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2025-01-11T14:38:26,990 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46115-0x101696b4fe20003 connected 2025-01-11T14:38:26,990 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46115-0x101696b4fe20003, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2025-01-11T14:38:26,990 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2025-01-11T14:38:26,991 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2025-01-11T14:38:26,991 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.cache 2025-01-11T14:38:26,991 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2025-01-11T14:38:26,991 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2025-01-11T14:38:26,991 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2025-01-11T14:38:26,992 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2025-01-11T14:38:26,992 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-11T14:38:26,992 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2025-01-11T14:38:26,993 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46115-0x101696b4fe20003, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2025-01-11T14:38:26,993 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2025-01-11T14:38:26,994 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2025-01-11T14:38:26,994 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46115-0x101696b4fe20003, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2025-01-11T14:38:26,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46115 2025-01-11T14:38:26,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46115 2025-01-11T14:38:26,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46115 2025-01-11T14:38:26,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46115 2025-01-11T14:38:26,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46115 2025-01-11T14:38:26,997 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2025-01-11T14:38:26,997 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer(746): ClusterId : 0fc479bb-18e4-4d64-aec5-40e5b85cc15a 2025-01-11T14:38:26,997 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2025-01-11T14:38:26,998 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-11T14:38:26,998 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2025-01-11T14:38:26,998 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.persistence is complete. 2025-01-11T14:38:27,000 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2025-01-11T14:38:27,000 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2025-01-11T14:38:27,002 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2025-01-11T14:38:27,003 DEBUG [RS:2;df3d9de72c82:46115 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76981071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=df3d9de72c82/172.17.0.2:0 2025-01-11T14:38:27,015 DEBUG [RS:2;df3d9de72c82:46115 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;df3d9de72c82:46115 2025-01-11T14:38:27,016 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2025-01-11T14:38:27,016 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2025-01-11T14:38:27,016 DEBUG [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer(832): About to register with Master. 2025-01-11T14:38:27,017 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer(2659): reportForDuty to master=df3d9de72c82,43135,1736606303576 with port=46115, startcode=1736606306983 2025-01-11T14:38:27,017 DEBUG [RS:2;df3d9de72c82:46115 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2025-01-11T14:38:27,019 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58247, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2025-01-11T14:38:27,020 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43135 {}] master.ServerManager(363): Checking decommissioned status of RegionServer df3d9de72c82,46115,1736606306983 2025-01-11T14:38:27,020 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43135 {}] master.ServerManager(517): Registering regionserver=df3d9de72c82,46115,1736606306983 2025-01-11T14:38:27,021 DEBUG [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a 2025-01-11T14:38:27,021 DEBUG [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2025-01-11T14:38:27,021 DEBUG [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2025-01-11T14:38:27,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2025-01-11T14:38:27,023 DEBUG [RS:2;df3d9de72c82:46115 {}] zookeeper.ZKUtil(111): regionserver:46115-0x101696b4fe20003, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/df3d9de72c82,46115,1736606306983 2025-01-11T14:38:27,023 WARN [RS:2;df3d9de72c82:46115 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2025-01-11T14:38:27,024 INFO [RS:2;df3d9de72c82:46115 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2025-01-11T14:38:27,024 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [df3d9de72c82,46115,1736606306983] 2025-01-11T14:38:27,024 DEBUG [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,46115,1736606306983 2025-01-11T14:38:27,030 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2025-01-11T14:38:27,031 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:27,031 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => f1c63d9675757a36716ce777981a4e99, NAME => 'testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99.', STARTKEY => '', ENDKEY => ''} 2025-01-11T14:38:27,032 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1736606304763 f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,032 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2025-01-11T14:38:27,032 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2025-01-11T14:38:27,032 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,032 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,032 INFO [RS:2;df3d9de72c82:46115 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2025-01-11T14:38:27,032 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:27,033 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2025-01-11T14:38:27,033 INFO [StoreOpener-f1c63d9675757a36716ce777981a4e99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,033 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2025-01-11T14:38:27,033 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:27,033 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:27,033 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:27,034 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:27,034 INFO [StoreOpener-f1c63d9675757a36716ce777981a4e99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1c63d9675757a36716ce777981a4e99 columnFamilyName family 2025-01-11T14:38:27,034 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:27,034 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:27,034 DEBUG [StoreOpener-f1c63d9675757a36716ce777981a4e99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2025-01-11T14:38:27,034 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/df3d9de72c82:0, corePoolSize=2, maxPoolSize=2 2025-01-11T14:38:27,034 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:27,034 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:27,034 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:27,034 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:27,034 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:27,034 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/df3d9de72c82:0, corePoolSize=1, maxPoolSize=1 2025-01-11T14:38:27,034 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/df3d9de72c82:0, corePoolSize=3, maxPoolSize=3 2025-01-11T14:38:27,035 DEBUG [RS:2;df3d9de72c82:46115 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/df3d9de72c82:0, corePoolSize=3, maxPoolSize=3 2025-01-11T14:38:27,035 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:27,035 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:27,035 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:27,035 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:27,035 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:27,035 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,46115,1736606306983-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2025-01-11T14:38:27,036 DEBUG [StoreFileOpener-f1c63d9675757a36716ce777981a4e99-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/family/98122ed62bc84df1811498ce18b48ad9, delay=958 ms 2025-01-11T14:38:27,036 DEBUG [StoreOpener-f1c63d9675757a36716ce777981a4e99-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/family/98122ed62bc84df1811498ce18b48ad9 2025-01-11T14:38:27,037 INFO [StoreOpener-f1c63d9675757a36716ce777981a4e99-1 {}] regionserver.HStore(327): Store=f1c63d9675757a36716ce777981a4e99/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2025-01-11T14:38:27,037 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,037 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,038 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,038 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,038 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,039 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,039 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened f1c63d9675757a36716ce777981a4e99; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69432224, jitterRate=0.03462076187133789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2025-01-11T14:38:27,039 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,041 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for f1c63d9675757a36716ce777981a4e99: Running coprocessor pre-open hook at 1736606307032Writing region info on filesystem at 1736606307032Initializing all the Stores at 1736606307032Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1736606307033 (+1 ms)Cleaning up temporary data from old regions at 1736606307038 (+5 ms)Running coprocessor post-open hooks at 1736606307039 (+1 ms)Region opened successfully at 1736606307041 (+2 ms) 2025-01-11T14:38:27,041 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99., pid=9, masterSystemTime=1736606307029 2025-01-11T14:38:27,044 DEBUG [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:27,044 INFO [RS_OPEN_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:27,045 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=f1c63d9675757a36716ce777981a4e99, regionState=OPEN, openSeqNum=10, regionLocation=df3d9de72c82,41723,1736606303692 2025-01-11T14:38:27,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1c63d9675757a36716ce777981a4e99, server=df3d9de72c82,41723,1736606303692 because future has completed 2025-01-11T14:38:27,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2025-01-11T14:38:27,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure f1c63d9675757a36716ce777981a4e99, server=df3d9de72c82,41723,1736606303692 in 173 msec 2025-01-11T14:38:27,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2025-01-11T14:38:27,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1736606304763, region=f1c63d9675757a36716ce777981a4e99, ASSIGN in 331 msec 2025-01-11T14:38:27,053 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server df3d9de72c82,44219,1736606303643 after splitting done 2025-01-11T14:38:27,053 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2025-01-11T14:38:27,053 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(168): Chore ScheduledChore name=df3d9de72c82,46115,1736606306983-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:27,053 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:27,053 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.Replication(171): df3d9de72c82,46115,1736606306983 started 2025-01-11T14:38:27,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure df3d9de72c82,44219,1736606303643, splitWal=true, meta=false in 352 msec 2025-01-11T14:38:27,070 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2025-01-11T14:38:27,070 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer(1482): Serving as df3d9de72c82,46115,1736606306983, RpcServer on df3d9de72c82/172.17.0.2:46115, sessionid=0x101696b4fe20003 2025-01-11T14:38:27,070 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2025-01-11T14:38:27,070 DEBUG [RS:2;df3d9de72c82:46115 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager df3d9de72c82,46115,1736606306983 2025-01-11T14:38:27,070 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'df3d9de72c82,46115,1736606306983' 2025-01-11T14:38:27,070 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2025-01-11T14:38:27,071 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2025-01-11T14:38:27,071 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2025-01-11T14:38:27,071 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2025-01-11T14:38:27,071 DEBUG [RS:2;df3d9de72c82:46115 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager df3d9de72c82,46115,1736606306983 2025-01-11T14:38:27,071 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'df3d9de72c82,46115,1736606306983' 2025-01-11T14:38:27,071 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2025-01-11T14:38:27,072 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2025-01-11T14:38:27,072 DEBUG [RS:2;df3d9de72c82:46115 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2025-01-11T14:38:27,073 INFO [RS:2;df3d9de72c82:46115 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2025-01-11T14:38:27,073 INFO [RS:2;df3d9de72c82:46115 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2025-01-11T14:38:27,173 INFO [RS:2;df3d9de72c82:46115 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2025-01-11T14:38:27,174 INFO [RS:2;df3d9de72c82:46115 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=df3d9de72c82%2C46115%2C1736606306983, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,46115,1736606306983, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs, maxLogs=55 2025-01-11T14:38:27,193 INFO [RS:2;df3d9de72c82:46115 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/WALs/df3d9de72c82,46115,1736606306983/df3d9de72c82%2C46115%2C1736606306983.1736606307175 2025-01-11T14:38:27,193 DEBUG [RS:2;df3d9de72c82:46115 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2025-01-11T14:38:27,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2025-01-11T14:38:27,570 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2025-01-11T14:38:27,570 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2025-01-11T14:38:27,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:27,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:27,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2025-01-11T14:38:27,571 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2025-01-11T14:38:27,571 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1096844908, stopped=false 2025-01-11T14:38:27,571 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=df3d9de72c82,43135,1736606303576 2025-01-11T14:38:27,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:27,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46115-0x101696b4fe20003, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:27,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2025-01-11T14:38:27,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:27,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:27,573 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2025-01-11T14:38:27,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46115-0x101696b4fe20003, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:27,573 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2025-01-11T14:38:27,573 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3321) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3292) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:686) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2025-01-11T14:38:27,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:27,574 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'df3d9de72c82,41723,1736606303692' ***** 2025-01-11T14:38:27,574 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2025-01-11T14:38:27,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:27,574 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'df3d9de72c82,46115,1736606306983' ***** 2025-01-11T14:38:27,574 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2025-01-11T14:38:27,574 INFO [RS:1;df3d9de72c82:41723 {}] bucket.BucketCache(1738): Disabling cache 2025-01-11T14:38:27,574 INFO [RS:2;df3d9de72c82:46115 {}] bucket.BucketCache(1738): Disabling cache 2025-01-11T14:38:27,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46115-0x101696b4fe20003, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:27,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2025-01-11T14:38:27,575 INFO [RS:2;df3d9de72c82:46115 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.persistence 2025-01-11T14:38:27,575 INFO [RS:1;df3d9de72c82:41723 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00162d31-c80b-da31-05da-0b8da8f9eb40/bucket.persistence 2025-01-11T14:38:27,575 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-11T14:38:27,575 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-11T14:38:27,575 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-11T14:38:27,575 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-11T14:38:27,575 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-11T14:38:27,575 INFO [RS:2;df3d9de72c82:46115 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2025-01-11T14:38:27,575 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-11T14:38:27,575 INFO [RS:1;df3d9de72c82:41723 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2025-01-11T14:38:27,575 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-11T14:38:27,575 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-11T14:38:27,585 DEBUG [RS:2;df3d9de72c82:46115 {}] bucket.BucketCache(1487): Thread RS:2;df3d9de72c82:46115 started persisting bucket cache to file 2025-01-11T14:38:27,585 DEBUG [RS:2;df3d9de72c82:46115 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2025-01-11T14:38:27,585 DEBUG [RS:1;df3d9de72c82:41723 {}] bucket.BucketCache(1487): Thread RS:1;df3d9de72c82:41723 started persisting bucket cache to file 2025-01-11T14:38:27,585 DEBUG [RS:2;df3d9de72c82:46115 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-11T14:38:27,585 DEBUG [RS:1;df3d9de72c82:41723 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2025-01-11T14:38:27,585 DEBUG [RS:1;df3d9de72c82:41723 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-11T14:38:27,589 DEBUG [RS:1;df3d9de72c82:41723 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-11T14:38:27,589 DEBUG [RS:2;df3d9de72c82:46115 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-11T14:38:27,589 DEBUG [RS:1;df3d9de72c82:41723 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1736606307585 2025-01-11T14:38:27,589 DEBUG [RS:2;df3d9de72c82:46115 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1736606307585 2025-01-11T14:38:27,589 DEBUG [RS:2;df3d9de72c82:46115 {}] bucket.BucketCache(1509): Thread RS:2;df3d9de72c82:46115 finished persisting bucket cache to file, renaming 2025-01-11T14:38:27,589 DEBUG [RS:1;df3d9de72c82:41723 {}] bucket.BucketCache(1509): Thread RS:1;df3d9de72c82:41723 finished persisting bucket cache to file, renaming 2025-01-11T14:38:27,589 WARN [RS:1;df3d9de72c82:41723 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2025-01-11T14:38:27,589 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.HeapMemoryManager(224): Stopping 2025-01-11T14:38:27,589 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HeapMemoryManager(224): Stopping 2025-01-11T14:38:27,589 INFO [RS:1;df3d9de72c82:41723 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2025-01-11T14:38:27,589 INFO [RS:2;df3d9de72c82:46115 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2025-01-11T14:38:27,590 INFO [RS:1;df3d9de72c82:41723 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2025-01-11T14:38:27,589 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2025-01-11T14:38:27,589 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2025-01-11T14:38:27,590 INFO [RS:2;df3d9de72c82:46115 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2025-01-11T14:38:27,590 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(3091): Received CLOSE for f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,590 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer(959): stopping server df3d9de72c82,46115,1736606306983 2025-01-11T14:38:27,590 INFO [RS:2;df3d9de72c82:46115 {}] hbase.HBaseServerBase(457): Close async cluster connection 2025-01-11T14:38:27,590 INFO [RS:2;df3d9de72c82:46115 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;df3d9de72c82:46115. 2025-01-11T14:38:27,590 DEBUG [RS:2;df3d9de72c82:46115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:459) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2025-01-11T14:38:27,590 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(959): stopping server df3d9de72c82,41723,1736606303692 2025-01-11T14:38:27,590 DEBUG [RS:2;df3d9de72c82:46115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:27,590 INFO [RS:1;df3d9de72c82:41723 {}] hbase.HBaseServerBase(457): Close async cluster connection 2025-01-11T14:38:27,590 INFO [RS:1;df3d9de72c82:41723 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;df3d9de72c82:41723. 2025-01-11T14:38:27,590 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer(976): stopping server df3d9de72c82,46115,1736606306983; all regions closed. 2025-01-11T14:38:27,590 DEBUG [RS:1;df3d9de72c82:41723 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:459) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2025-01-11T14:38:27,590 DEBUG [RS:1;df3d9de72c82:41723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:27,590 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2025-01-11T14:38:27,590 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2025-01-11T14:38:27,590 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2025-01-11T14:38:27,590 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f1c63d9675757a36716ce777981a4e99, disabling compactions & flushes 2025-01-11T14:38:27,590 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2025-01-11T14:38:27,590 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2025-01-11T14:38:27,590 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:27,590 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:27,591 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. after waiting 0 ms 2025-01-11T14:38:27,591 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:27,591 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2025-01-11T14:38:27,591 DEBUG [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(1325): Online Regions={f1c63d9675757a36716ce777981a4e99=testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99., 1588230740=hbase:meta,,1.1588230740} 2025-01-11T14:38:27,591 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2025-01-11T14:38:27,591 DEBUG [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f1c63d9675757a36716ce777981a4e99 2025-01-11T14:38:27,591 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2025-01-11T14:38:27,591 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2025-01-11T14:38:27,591 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2025-01-11T14:38:27,591 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2025-01-11T14:38:27,591 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2025-01-11T14:38:27,591 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2025-01-11T14:38:27,592 DEBUG [StoreFileCloser-f1c63d9675757a36716ce777981a4e99-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/family/98122ed62bc84df1811498ce18b48ad9 2025-01-11T14:38:27,592 DEBUG [RS:2;df3d9de72c82:46115 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs 2025-01-11T14:38:27,592 INFO [RS:2;df3d9de72c82:46115 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL df3d9de72c82%2C46115%2C1736606306983:(num 1736606307175) 2025-01-11T14:38:27,592 DEBUG [RS:2;df3d9de72c82:46115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:27,592 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.LeaseManager(133): Closed leases 2025-01-11T14:38:27,593 INFO [RS:2;df3d9de72c82:46115 {}] hbase.HBaseServerBase(440): Shutdown chores and chore service 2025-01-11T14:38:27,593 INFO [RS:2;df3d9de72c82:46115 {}] hbase.ChoreService(370): Chore service for: regionserver/df3d9de72c82:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2025-01-11T14:38:27,593 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2025-01-11T14:38:27,593 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2025-01-11T14:38:27,593 INFO [regionserver/df3d9de72c82:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2025-01-11T14:38:27,593 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2025-01-11T14:38:27,593 INFO [RS:2;df3d9de72c82:46115 {}] hbase.HBaseServerBase(450): Shutdown executor service 2025-01-11T14:38:27,593 INFO [RS:2;df3d9de72c82:46115 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46115 2025-01-11T14:38:27,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2025-01-11T14:38:27,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46115-0x101696b4fe20003, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/df3d9de72c82,46115,1736606306983 2025-01-11T14:38:27,595 INFO [RS:2;df3d9de72c82:46115 {}] hbase.HBaseServerBase(481): Close zookeeper 2025-01-11T14:38:27,597 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [df3d9de72c82,46115,1736606306983] 2025-01-11T14:38:27,597 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/default/testBlockEvictionOnGracefulStop1736606304763/f1c63d9675757a36716ce777981a4e99/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2025-01-11T14:38:27,597 INFO [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:27,597 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f1c63d9675757a36716ce777981a4e99: Waiting for close lock at 1736606307590Running coprocessor pre-close hooks at 1736606307590Disabling compacts and flushes for region at 1736606307590Disabling writes for close at 1736606307591 (+1 ms)Writing region close event to WAL at 1736606307592 (+1 ms)Running coprocessor post-close hooks at 1736606307597 (+5 ms)Closed at 1736606307597 2025-01-11T14:38:27,598 DEBUG [RS_CLOSE_REGION-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99. 2025-01-11T14:38:27,598 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/df3d9de72c82,46115,1736606306983 already deleted, retry=false 2025-01-11T14:38:27,598 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; df3d9de72c82,46115,1736606306983 expired; onlineServers=1 2025-01-11T14:38:27,603 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 92d2a6563e204b34ad706b23e6c65ee0 2025-01-11T14:38:27,617 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/info/92d2a6563e204b34ad706b23e6c65ee0 is 209, key is testBlockEvictionOnGracefulStop1736606304763,,1736606304775.f1c63d9675757a36716ce777981a4e99./info:regioninfo/1736606307045/Put/seqid=0 2025-01-11T14:38:27,617 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/info/92d2a6563e204b34ad706b23e6c65ee0 2025-01-11T14:38:27,627 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 22c84948a1e74f0b9d9b48ba99c084bd 2025-01-11T14:38:27,636 INFO [regionserver/df3d9de72c82:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2025-01-11T14:38:27,644 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/ns/22c84948a1e74f0b9d9b48ba99c084bd is 43, key is default/ns:d/1736606304671/Put/seqid=0 2025-01-11T14:38:27,645 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/ns/22c84948a1e74f0b9d9b48ba99c084bd 2025-01-11T14:38:27,656 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ccc4ea5d6255441aa2945a94649dcf29 2025-01-11T14:38:27,661 INFO [regionserver/df3d9de72c82:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2025-01-11T14:38:27,672 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/table/ccc4ea5d6255441aa2945a94649dcf29 is 80, key is testBlockEvictionOnGracefulStop1736606304763/table:state/1736606305179/Put/seqid=0 2025-01-11T14:38:27,673 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/table/ccc4ea5d6255441aa2945a94649dcf29 2025-01-11T14:38:27,676 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/info/92d2a6563e204b34ad706b23e6c65ee0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/info/92d2a6563e204b34ad706b23e6c65ee0 2025-01-11T14:38:27,677 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/info/92d2a6563e204b34ad706b23e6c65ee0, delay=922 ms 2025-01-11T14:38:27,679 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/info/92d2a6563e204b34ad706b23e6c65ee0, entries=14, sequenceid=13, filesize=7.5 K 2025-01-11T14:38:27,679 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/ns/22c84948a1e74f0b9d9b48ba99c084bd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/ns/22c84948a1e74f0b9d9b48ba99c084bd 2025-01-11T14:38:27,681 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/ns/22c84948a1e74f0b9d9b48ba99c084bd, delay=993 ms 2025-01-11T14:38:27,682 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/ns/22c84948a1e74f0b9d9b48ba99c084bd, entries=2, sequenceid=13, filesize=5.0 K 2025-01-11T14:38:27,683 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/.tmp/table/ccc4ea5d6255441aa2945a94649dcf29 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/table/ccc4ea5d6255441aa2945a94649dcf29 2025-01-11T14:38:27,684 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/table/ccc4ea5d6255441aa2945a94649dcf29, delay=911 ms 2025-01-11T14:38:27,685 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/table/ccc4ea5d6255441aa2945a94649dcf29, entries=2, sequenceid=13, filesize=5.3 K 2025-01-11T14:38:27,686 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=13, compaction requested=false 2025-01-11T14:38:27,687 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/info/92d2a6563e204b34ad706b23e6c65ee0 2025-01-11T14:38:27,687 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/ns/22c84948a1e74f0b9d9b48ba99c084bd 2025-01-11T14:38:27,688 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/table/ccc4ea5d6255441aa2945a94649dcf29 2025-01-11T14:38:27,692 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2025-01-11T14:38:27,693 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2025-01-11T14:38:27,693 INFO [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2025-01-11T14:38:27,693 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1736606307591Running coprocessor pre-close hooks at 1736606307591Disabling compacts and flushes for region at 1736606307591Disabling writes for close at 1736606307591Obtaining lock to block concurrent updates at 1736606307591Preparing flush snapshotting stores in 1588230740 at 1736606307591Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1736606307592 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1736606307592Flushing 1588230740/info: creating writer at 1736606307592Flushing 1588230740/info: appending metadata at 1736606307616 (+24 ms)Flushing 1588230740/info: closing flushed file at 1736606307616Flushing 1588230740/ns: creating writer at 1736606307620 (+4 ms)Flushing 1588230740/ns: appending metadata at 1736606307643 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1736606307643Flushing 1588230740/table: creating writer at 1736606307649 (+6 ms)Flushing 1588230740/table: appending metadata at 1736606307672 (+23 ms)Flushing 1588230740/table: closing flushed file at 1736606307672Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27d725f3: reopening flushed file at 1736606307676 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3aed9b5c: reopening flushed file at 1736606307679 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ef28d1b: reopening flushed file at 1736606307682 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=13, compaction requested=false at 1736606307686 (+4 ms)Writing region close event to WAL at 1736606307688 (+2 ms)Running coprocessor post-close hooks at 1736606307692 (+4 ms)Closed at 1736606307693 (+1 ms) 2025-01-11T14:38:27,693 DEBUG [RS_CLOSE_META-regionserver/df3d9de72c82:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2025-01-11T14:38:27,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46115-0x101696b4fe20003, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:27,697 INFO [RS:2;df3d9de72c82:46115 {}] hbase.HBaseServerBase(488): Close table descriptors 2025-01-11T14:38:27,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46115-0x101696b4fe20003, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:27,697 INFO [RS:2;df3d9de72c82:46115 {}] regionserver.HRegionServer(1031): Exiting; stopping=df3d9de72c82,46115,1736606306983; zookeeper connection closed. 2025-01-11T14:38:27,697 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@519a2a42 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@519a2a42 2025-01-11T14:38:27,791 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(976): stopping server df3d9de72c82,41723,1736606303692; all regions closed. 2025-01-11T14:38:27,793 DEBUG [RS:1;df3d9de72c82:41723 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs 2025-01-11T14:38:27,793 INFO [RS:1;df3d9de72c82:41723 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL df3d9de72c82%2C41723%2C1736606303692.meta:.meta(num 1736606304606) 2025-01-11T14:38:27,795 DEBUG [RS:1;df3d9de72c82:41723 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/oldWALs 2025-01-11T14:38:27,795 INFO [RS:1;df3d9de72c82:41723 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL df3d9de72c82%2C41723%2C1736606303692:(num 1736606304407) 2025-01-11T14:38:27,795 DEBUG [RS:1;df3d9de72c82:41723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2025-01-11T14:38:27,795 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.LeaseManager(133): Closed leases 2025-01-11T14:38:27,795 INFO [RS:1;df3d9de72c82:41723 {}] hbase.HBaseServerBase(440): Shutdown chores and chore service 2025-01-11T14:38:27,795 INFO [RS:1;df3d9de72c82:41723 {}] hbase.ChoreService(370): Chore service for: regionserver/df3d9de72c82:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2025-01-11T14:38:27,795 INFO [RS:1;df3d9de72c82:41723 {}] hbase.HBaseServerBase(450): Shutdown executor service 2025-01-11T14:38:27,795 INFO [regionserver/df3d9de72c82:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2025-01-11T14:38:27,796 INFO [RS:1;df3d9de72c82:41723 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41723 2025-01-11T14:38:27,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/df3d9de72c82,41723,1736606303692 2025-01-11T14:38:27,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2025-01-11T14:38:27,798 INFO [RS:1;df3d9de72c82:41723 {}] hbase.HBaseServerBase(481): Close zookeeper 2025-01-11T14:38:27,799 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [df3d9de72c82,41723,1736606303692] 2025-01-11T14:38:27,800 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/df3d9de72c82,41723,1736606303692 already deleted, retry=false 2025-01-11T14:38:27,800 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; df3d9de72c82,41723,1736606303692 expired; onlineServers=0 2025-01-11T14:38:27,800 INFO [RegionServerTracker-0 {}] master.HMaster(3336): ***** STOPPING master 'df3d9de72c82,43135,1736606303576' ***** 2025-01-11T14:38:27,800 INFO [RegionServerTracker-0 {}] master.HMaster(3338): STOPPED: Cluster shutdown set; onlineServer=0 2025-01-11T14:38:27,800 INFO [M:0;df3d9de72c82:43135 {}] hbase.HBaseServerBase(457): Close async cluster connection 2025-01-11T14:38:27,800 INFO [M:0;df3d9de72c82:43135 {}] hbase.HBaseServerBase(440): Shutdown chores and chore service 2025-01-11T14:38:27,800 DEBUG [M:0;df3d9de72c82:43135 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2025-01-11T14:38:27,801 DEBUG [M:0;df3d9de72c82:43135 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2025-01-11T14:38:27,801 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2025-01-11T14:38:27,801 DEBUG [master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.large.0-1736606304120 {}] cleaner.HFileCleaner(306): Exit Thread[master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.large.0-1736606304120,5,FailOnTimeoutGroup] 2025-01-11T14:38:27,801 DEBUG [master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.small.0-1736606304120 {}] cleaner.HFileCleaner(306): Exit Thread[master/df3d9de72c82:0:becomeActiveMaster-HFileCleaner.small.0-1736606304120,5,FailOnTimeoutGroup] 2025-01-11T14:38:27,801 INFO [M:0;df3d9de72c82:43135 {}] hbase.ChoreService(370): Chore service for: master/df3d9de72c82:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2025-01-11T14:38:27,801 INFO [M:0;df3d9de72c82:43135 {}] hbase.HBaseServerBase(450): Shutdown executor service 2025-01-11T14:38:27,801 DEBUG [M:0;df3d9de72c82:43135 {}] master.HMaster(1806): Stopping service threads 2025-01-11T14:38:27,801 INFO [M:0;df3d9de72c82:43135 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2025-01-11T14:38:27,801 INFO [M:0;df3d9de72c82:43135 {}] procedure2.ProcedureExecutor(723): Stopping 2025-01-11T14:38:27,801 INFO [M:0;df3d9de72c82:43135 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2025-01-11T14:38:27,802 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2025-01-11T14:38:27,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2025-01-11T14:38:27,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2025-01-11T14:38:27,802 DEBUG [M:0;df3d9de72c82:43135 {}] zookeeper.ZKUtil(347): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2025-01-11T14:38:27,802 WARN [M:0;df3d9de72c82:43135 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2025-01-11T14:38:27,803 INFO [M:0;df3d9de72c82:43135 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/.lastflushedseqids 2025-01-11T14:38:27,810 INFO [M:0;df3d9de72c82:43135 {}] assignment.AssignmentManager(395): Stopping assignment manager 2025-01-11T14:38:27,810 INFO [M:0;df3d9de72c82:43135 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2025-01-11T14:38:27,810 DEBUG [M:0;df3d9de72c82:43135 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2025-01-11T14:38:27,810 INFO [M:0;df3d9de72c82:43135 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:27,810 DEBUG [M:0;df3d9de72c82:43135 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:27,810 DEBUG [M:0;df3d9de72c82:43135 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2025-01-11T14:38:27,810 DEBUG [M:0;df3d9de72c82:43135 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:27,811 INFO [M:0;df3d9de72c82:43135 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.81 KB heapSize=41.05 KB 2025-01-11T14:38:27,822 DEBUG [M:0;df3d9de72c82:43135 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1bd3b9bf67dc41eaa6c20caf6f6baf4e 2025-01-11T14:38:27,838 DEBUG [M:0;df3d9de72c82:43135 {}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bd3b9bf67dc41eaa6c20caf6f6baf4e is 82, key is hbase:meta,,1/info:regioninfo/1736606304652/Put/seqid=0 2025-01-11T14:38:27,839 INFO [M:0;df3d9de72c82:43135 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bd3b9bf67dc41eaa6c20caf6f6baf4e 2025-01-11T14:38:27,847 DEBUG [M:0;df3d9de72c82:43135 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 270123c15f2046d8b01b46c23725cc9b 2025-01-11T14:38:27,861 DEBUG [M:0;df3d9de72c82:43135 {}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/270123c15f2046d8b01b46c23725cc9b is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1736606305184/Put/seqid=0 2025-01-11T14:38:27,861 INFO [M:0;df3d9de72c82:43135 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.08 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/270123c15f2046d8b01b46c23725cc9b 2025-01-11T14:38:27,870 DEBUG [M:0;df3d9de72c82:43135 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bd154f76f86648449eaffd00fe53e450 2025-01-11T14:38:27,884 DEBUG [M:0;df3d9de72c82:43135 {}] hfile.HFileWriterImpl(821): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bd154f76f86648449eaffd00fe53e450 is 69, key is df3d9de72c82,41723,1736606303692/rs:state/1736606304200/Put/seqid=0 2025-01-11T14:38:27,884 INFO [M:0;df3d9de72c82:43135 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bd154f76f86648449eaffd00fe53e450 2025-01-11T14:38:27,886 INFO [M:0;df3d9de72c82:43135 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bd154f76f86648449eaffd00fe53e450 2025-01-11T14:38:27,886 DEBUG [M:0;df3d9de72c82:43135 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bd3b9bf67dc41eaa6c20caf6f6baf4e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1bd3b9bf67dc41eaa6c20caf6f6baf4e 2025-01-11T14:38:27,888 INFO [M:0;df3d9de72c82:43135 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1bd3b9bf67dc41eaa6c20caf6f6baf4e, entries=8, sequenceid=83, filesize=5.5 K 2025-01-11T14:38:27,888 DEBUG [M:0;df3d9de72c82:43135 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/270123c15f2046d8b01b46c23725cc9b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/270123c15f2046d8b01b46c23725cc9b 2025-01-11T14:38:27,890 INFO [M:0;df3d9de72c82:43135 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/270123c15f2046d8b01b46c23725cc9b, entries=9, sequenceid=83, filesize=6.4 K 2025-01-11T14:38:27,890 DEBUG [M:0;df3d9de72c82:43135 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bd154f76f86648449eaffd00fe53e450 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bd154f76f86648449eaffd00fe53e450 2025-01-11T14:38:27,891 INFO [M:0;df3d9de72c82:43135 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bd154f76f86648449eaffd00fe53e450 2025-01-11T14:38:27,892 INFO [M:0;df3d9de72c82:43135 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d76fc405-8024-57af-2e83-079891885de0/6e2b762b-d1bf-038b-d786-b4aa0d43b37a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bd154f76f86648449eaffd00fe53e450, entries=3, sequenceid=83, filesize=5.3 K 2025-01-11T14:38:27,892 INFO [M:0;df3d9de72c82:43135 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=83, compaction requested=false 2025-01-11T14:38:27,894 INFO [M:0;df3d9de72c82:43135 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2025-01-11T14:38:27,894 DEBUG [M:0;df3d9de72c82:43135 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1736606307810Disabling compacts and flushes for region at 1736606307810Disabling writes for close at 1736606307810Obtaining lock to block concurrent updates at 1736606307811 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1736606307811Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33602, getHeapSize=41976, getOffHeapSize=0, getCellsCount=98 at 1736606307811Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1736606307811Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1736606307811Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1736606307837 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1736606307837Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1736606307840 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1736606307860 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1736606307860Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1736606307863 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1736606307883 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1736606307883Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17f4387: reopening flushed file at 1736606307886 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6161db97: reopening flushed file at 1736606307888 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@754320b0: reopening flushed file at 1736606307890 (+2 ms)Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=83, compaction requested=false at 1736606307892 (+2 ms)Writing region close event to WAL at 1736606307894 (+2 ms)Closed at 1736606307894 2025-01-11T14:38:27,895 INFO [M:0;df3d9de72c82:43135 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2025-01-11T14:38:27,895 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2025-01-11T14:38:27,895 INFO [M:0;df3d9de72c82:43135 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43135 2025-01-11T14:38:27,895 INFO [M:0;df3d9de72c82:43135 {}] hbase.HBaseServerBase(481): Close zookeeper 2025-01-11T14:38:27,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:27,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41723-0x101696b4fe20002, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:27,899 INFO [RS:1;df3d9de72c82:41723 {}] hbase.HBaseServerBase(488): Close table descriptors 2025-01-11T14:38:27,899 INFO [RS:1;df3d9de72c82:41723 {}] regionserver.HRegionServer(1031): Exiting; stopping=df3d9de72c82,41723,1736606303692; zookeeper connection closed. 2025-01-11T14:38:27,899 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5f6f8de4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5f6f8de4 2025-01-11T14:38:27,900 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2025-01-11T14:38:27,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:27,998 INFO [M:0;df3d9de72c82:43135 {}] hbase.HBaseServerBase(488): Close table descriptors 2025-01-11T14:38:27,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43135-0x101696b4fe20000, quorum=127.0.0.1:64262, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2025-01-11T14:38:28,000 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2025-01-11T14:38:28,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2025-01-11T14:38:28,001 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-11T14:38:28,012 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=246 (was 246), ProcessCount=11 (was 11), AvailableMemoryMB=4841 (was 5195)