2024-11-12 15:23:45,578 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-12 15:23:45,608 main DEBUG Took 0.027542 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 15:23:45,609 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 15:23:45,609 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 15:23:45,611 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 15:23:45,613 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,622 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 15:23:45,637 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,640 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,640 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,641 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,641 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,643 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,644 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,644 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,645 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,646 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,647 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,648 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,649 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,649 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,650 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,650 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,651 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,651 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,652 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,653 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,653 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,654 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:23:45,655 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,655 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 15:23:45,657 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:23:45,659 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 15:23:45,661 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 15:23:45,662 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 15:23:45,664 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 15:23:45,664 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 15:23:45,675 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 15:23:45,680 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 15:23:45,683 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 15:23:45,683 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 15:23:45,684 main DEBUG createAppenders(={Console}) 2024-11-12 15:23:45,685 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-12 15:23:45,686 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-12 15:23:45,686 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-12 15:23:45,687 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 15:23:45,688 main DEBUG OutputStream closed 2024-11-12 15:23:45,688 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 15:23:45,688 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 15:23:45,689 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-12 15:23:45,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 15:23:45,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 15:23:45,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 15:23:45,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 15:23:45,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 15:23:45,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 15:23:45,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 15:23:45,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 15:23:45,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 15:23:45,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 15:23:45,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 15:23:45,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 15:23:45,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 15:23:45,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 15:23:45,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 15:23:45,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 15:23:45,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 15:23:45,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 15:23:45,792 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 15:23:45,792 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-12 15:23:45,792 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 15:23:45,797 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-12T15:23:45,814 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-12 15:23:45,817 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 15:23:45,818 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T15:23:46,057 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71 2024-11-12T15:23:46,079 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=342, ProcessCount=11, AvailableMemoryMB=9665 2024-11-12T15:23:46,173 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T15:23:46,253 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T15:23:46,266 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/cluster_d7bed061-f9af-a2b5-dceb-31eb2a898e4e, deleteOnExit=true 2024-11-12T15:23:46,341 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/cluster_d7bed061-f9af-a2b5-dceb-31eb2a898e4e/zookeeper_0, clientPort=51493, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/cluster_d7bed061-f9af-a2b5-dceb-31eb2a898e4e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/cluster_d7bed061-f9af-a2b5-dceb-31eb2a898e4e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T15:23:46,352 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51493 2024-11-12T15:23:46,353 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T15:23:46,426 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85 with version=8 2024-11-12T15:23:46,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/hbase-staging 2024-11-12T15:23:46,515 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T15:23:46,786 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a6f201fa5419:0 server-side Connection retries=45 2024-11-12T15:23:46,794 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T15:23:46,795 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T15:23:46,803 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T15:23:46,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T15:23:46,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T15:23:46,936 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T15:23:47,028 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T15:23:47,081 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T15:23:47,093 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T15:23:47,098 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T15:23:47,142 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 34321 (auto-detected) 2024-11-12T15:23:47,143 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T15:23:47,171 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36173 2024-11-12T15:23:47,195 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36173 connecting to ZooKeeper ensemble=127.0.0.1:51493 2024-11-12T15:23:47,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:361730x0, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T15:23:47,344 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36173-0x1012fa53cbf0000 connected 2024-11-12T15:23:47,445 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T15:23:47,447 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T15:23:47,456 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T15:23:47,461 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85, hbase.cluster.distributed=false 2024-11-12T15:23:47,490 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T15:23:47,495 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36173 2024-11-12T15:23:47,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36173 2024-11-12T15:23:47,499 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36173 2024-11-12T15:23:47,500 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36173 2024-11-12T15:23:47,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36173 2024-11-12T15:23:47,585 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a6f201fa5419:0 server-side Connection retries=45 2024-11-12T15:23:47,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T15:23:47,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T15:23:47,588 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T15:23:47,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T15:23:47,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T15:23:47,590 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T15:23:47,593 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T15:23:47,594 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33465 2024-11-12T15:23:47,596 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33465 connecting to ZooKeeper ensemble=127.0.0.1:51493 2024-11-12T15:23:47,597 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T15:23:47,598 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T15:23:47,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334650x0, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T15:23:47,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T15:23:47,610 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33465-0x1012fa53cbf0001 connected 2024-11-12T15:23:47,616 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T15:23:47,629 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/bucket.cache 2024-11-12T15:23:47,635 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T15:23:47,636 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T15:23:47,637 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-12T15:23:47,639 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T15:23:47,639 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T15:23:47,644 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T15:23:47,647 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T15:23:47,647 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/bucket.persistence is complete. 2024-11-12T15:23:47,649 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T15:23:47,652 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T15:23:47,659 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T15:23:47,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33465 2024-11-12T15:23:47,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33465 2024-11-12T15:23:47,665 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33465 2024-11-12T15:23:47,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33465 2024-11-12T15:23:47,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33465 2024-11-12T15:23:47,683 DEBUG [M:0;a6f201fa5419:36173 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a6f201fa5419:36173 2024-11-12T15:23:47,684 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a6f201fa5419,36173,1731425026619 2024-11-12T15:23:47,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T15:23:47,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T15:23:47,705 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a6f201fa5419,36173,1731425026619 2024-11-12T15:23:47,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T15:23:47,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:47,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:47,737 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T15:23:47,738 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a6f201fa5419,36173,1731425026619 from backup master directory 2024-11-12T15:23:47,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T15:23:47,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a6f201fa5419,36173,1731425026619 2024-11-12T15:23:47,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T15:23:47,747 WARN [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T15:23:47,747 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a6f201fa5419,36173,1731425026619 2024-11-12T15:23:47,750 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T15:23:47,751 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T15:23:47,759 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/hbase.id] with ID: a3493279-838d-4391-8325-94791f2fe6e7 2024-11-12T15:23:47,760 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/.tmp/hbase.id 2024-11-12T15:23:47,765 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/hbase.id] 2024-11-12T15:23:47,807 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T15:23:47,814 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T15:23:47,834 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-12T15:23:47,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:47,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:47,878 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:23:47,881 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T15:23:47,922 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T15:23:47,927 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T15:23:47,973 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store 2024-11-12T15:23:47,992 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T15:23:47,996 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:23:47,997 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T15:23:47,997 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:23:47,997 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:23:47,999 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T15:23:47,999 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:23:47,999 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:23:48,000 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731425027997Disabling compacts and flushes for region at 1731425027997Disabling writes for close at 1731425027999 (+2 ms)Writing region close event to WAL at 1731425027999Closed at 1731425027999 2024-11-12T15:23:48,000 WARN [master/a6f201fa5419:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/.initializing 2024-11-12T15:23:48,000 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/WALs/a6f201fa5419,36173,1731425026619 2024-11-12T15:23:48,012 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T15:23:48,024 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a6f201fa5419%2C36173%2C1731425026619, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/WALs/a6f201fa5419,36173,1731425026619, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/oldWALs, maxLogs=10 2024-11-12T15:23:48,060 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/WALs/a6f201fa5419,36173,1731425026619/a6f201fa5419%2C36173%2C1731425026619.1731425028028 2024-11-12T15:23:48,061 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T15:23:48,061 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:23:48,062 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:23:48,065 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,066 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,100 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,125 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T15:23:48,129 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:48,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:23:48,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T15:23:48,136 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:48,137 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:23:48,137 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T15:23:48,142 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:48,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:23:48,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T15:23:48,148 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:48,148 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:23:48,149 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,152 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,152 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,155 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,156 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,158 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T15:23:48,160 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:23:48,166 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T15:23:48,167 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70980669, jitterRate=0.057694390416145325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T15:23:48,178 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731425028078Initializing all the Stores at 1731425028078Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731425028079 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731425028080 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731425028080Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731425028080Cleaning up temporary data from old regions at 1731425028156 (+76 ms)Region opened successfully at 1731425028177 (+21 ms) 2024-11-12T15:23:48,180 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T15:23:48,212 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ecc6d65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a6f201fa5419/172.17.0.2:0 2024-11-12T15:23:48,245 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T15:23:48,258 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T15:23:48,258 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T15:23:48,261 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T15:23:48,262 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T15:23:48,268 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-12T15:23:48,269 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T15:23:48,296 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T15:23:48,306 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T15:23:48,356 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T15:23:48,359 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T15:23:48,361 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T15:23:48,367 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T15:23:48,369 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T15:23:48,373 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T15:23:48,386 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T15:23:48,388 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T15:23:48,398 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T15:23:48,415 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T15:23:48,428 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T15:23:48,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T15:23:48,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T15:23:48,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:48,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:48,453 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a6f201fa5419,36173,1731425026619, sessionid=0x1012fa53cbf0000, setting cluster-up flag (Was=false) 2024-11-12T15:23:48,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:48,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:48,514 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T15:23:48,517 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a6f201fa5419,36173,1731425026619 2024-11-12T15:23:48,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:48,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:48,567 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T15:23:48,572 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a6f201fa5419,36173,1731425026619 2024-11-12T15:23:48,579 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T15:23:48,652 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T15:23:48,662 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T15:23:48,668 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T15:23:48,671 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(746): ClusterId : a3493279-838d-4391-8325-94791f2fe6e7 2024-11-12T15:23:48,674 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T15:23:48,675 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a6f201fa5419,36173,1731425026619 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T15:23:48,685 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T15:23:48,685 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T15:23:48,685 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a6f201fa5419:0, corePoolSize=5, maxPoolSize=5 2024-11-12T15:23:48,685 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a6f201fa5419:0, corePoolSize=5, maxPoolSize=5 2024-11-12T15:23:48,686 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a6f201fa5419:0, corePoolSize=5, maxPoolSize=5 2024-11-12T15:23:48,686 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a6f201fa5419:0, corePoolSize=5, maxPoolSize=5 2024-11-12T15:23:48,686 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a6f201fa5419:0, corePoolSize=10, maxPoolSize=10 2024-11-12T15:23:48,686 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,687 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a6f201fa5419:0, corePoolSize=2, maxPoolSize=2 2024-11-12T15:23:48,687 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,688 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731425058688 2024-11-12T15:23:48,689 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T15:23:48,690 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T15:23:48,691 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T15:23:48,692 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T15:23:48,693 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T15:23:48,694 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T15:23:48,694 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T15:23:48,694 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T15:23:48,696 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T15:23:48,697 DEBUG [RS:0;a6f201fa5419:33465 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c0fc44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a6f201fa5419/172.17.0.2:0 2024-11-12T15:23:48,697 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:48,697 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T15:23:48,696 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,703 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T15:23:48,704 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T15:23:48,705 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T15:23:48,709 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T15:23:48,710 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T15:23:48,712 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a6f201fa5419:0:becomeActiveMaster-HFileCleaner.large.0-1731425028711,5,FailOnTimeoutGroup] 2024-11-12T15:23:48,713 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a6f201fa5419:0:becomeActiveMaster-HFileCleaner.small.0-1731425028712,5,FailOnTimeoutGroup] 2024-11-12T15:23:48,714 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,714 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T15:23:48,715 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,716 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,718 DEBUG [RS:0;a6f201fa5419:33465 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a6f201fa5419:33465 2024-11-12T15:23:48,721 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T15:23:48,722 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T15:23:48,722 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T15:23:48,722 DEBUG [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T15:23:48,722 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85 2024-11-12T15:23:48,725 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(2659): reportForDuty to master=a6f201fa5419,36173,1731425026619 with port=33465, startcode=1731425027553 2024-11-12T15:23:48,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:23:48,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T15:23:48,735 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T15:23:48,735 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:48,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:23:48,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T15:23:48,738 DEBUG [RS:0;a6f201fa5419:33465 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T15:23:48,741 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T15:23:48,741 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:48,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:23:48,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T15:23:48,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T15:23:48,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:48,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:23:48,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T15:23:48,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T15:23:48,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:48,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:23:48,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T15:23:48,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740 2024-11-12T15:23:48,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740 2024-11-12T15:23:48,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T15:23:48,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T15:23:48,752 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T15:23:48,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T15:23:48,758 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T15:23:48,759 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68077462, jitterRate=0.014433234930038452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T15:23:48,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731425028731Initializing all the Stores at 1731425028731Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731425028731Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731425028731Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731425028731Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731425028732 (+1 ms)Cleaning up temporary data from old regions at 1731425028752 (+20 ms)Region opened successfully at 1731425028763 (+11 ms) 2024-11-12T15:23:48,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T15:23:48,763 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T15:23:48,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T15:23:48,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 1 ms 2024-11-12T15:23:48,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T15:23:48,765 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T15:23:48,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731425028763Disabling compacts and flushes for region at 1731425028763Disabling writes for close at 1731425028764 (+1 ms)Writing region close event to WAL at 1731425028765 (+1 ms)Closed at 1731425028765 2024-11-12T15:23:48,768 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T15:23:48,768 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T15:23:48,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T15:23:48,786 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T15:23:48,789 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T15:23:48,814 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46981, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T15:23:48,822 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36173 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a6f201fa5419,33465,1731425027553 2024-11-12T15:23:48,825 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36173 {}] master.ServerManager(517): Registering regionserver=a6f201fa5419,33465,1731425027553 2024-11-12T15:23:48,840 DEBUG [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85 2024-11-12T15:23:48,840 DEBUG [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T15:23:48,840 DEBUG [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T15:23:48,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T15:23:48,894 DEBUG [RS:0;a6f201fa5419:33465 {}] zookeeper.ZKUtil(111): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a6f201fa5419,33465,1731425027553 2024-11-12T15:23:48,895 WARN [RS:0;a6f201fa5419:33465 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T15:23:48,895 INFO [RS:0;a6f201fa5419:33465 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T15:23:48,895 DEBUG [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/WALs/a6f201fa5419,33465,1731425027553 2024-11-12T15:23:48,897 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a6f201fa5419,33465,1731425027553] 2024-11-12T15:23:48,919 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T15:23:48,938 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T15:23:48,940 WARN [a6f201fa5419:36173 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-12T15:23:48,945 INFO [RS:0;a6f201fa5419:33465 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T15:23:48,945 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,946 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T15:23:48,951 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T15:23:48,952 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,953 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,953 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,953 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,953 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,953 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,954 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a6f201fa5419:0, corePoolSize=2, maxPoolSize=2 2024-11-12T15:23:48,954 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,954 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,954 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,954 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,955 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,955 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a6f201fa5419:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:23:48,955 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a6f201fa5419:0, corePoolSize=3, maxPoolSize=3 2024-11-12T15:23:48,955 DEBUG [RS:0;a6f201fa5419:33465 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a6f201fa5419:0, corePoolSize=3, maxPoolSize=3 2024-11-12T15:23:48,957 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,957 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,957 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,957 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,957 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,958 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(168): Chore ScheduledChore name=a6f201fa5419,33465,1731425027553-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T15:23:48,979 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T15:23:48,981 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(168): Chore ScheduledChore name=a6f201fa5419,33465,1731425027553-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,982 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:48,982 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.Replication(171): a6f201fa5419,33465,1731425027553 started 2024-11-12T15:23:49,002 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:49,003 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(1482): Serving as a6f201fa5419,33465,1731425027553, RpcServer on a6f201fa5419/172.17.0.2:33465, sessionid=0x1012fa53cbf0001 2024-11-12T15:23:49,004 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T15:23:49,004 DEBUG [RS:0;a6f201fa5419:33465 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a6f201fa5419,33465,1731425027553 2024-11-12T15:23:49,004 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a6f201fa5419,33465,1731425027553' 2024-11-12T15:23:49,004 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T15:23:49,005 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T15:23:49,006 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T15:23:49,006 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T15:23:49,006 DEBUG [RS:0;a6f201fa5419:33465 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a6f201fa5419,33465,1731425027553 2024-11-12T15:23:49,007 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a6f201fa5419,33465,1731425027553' 2024-11-12T15:23:49,007 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T15:23:49,007 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T15:23:49,008 DEBUG [RS:0;a6f201fa5419:33465 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T15:23:49,008 INFO [RS:0;a6f201fa5419:33465 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T15:23:49,008 INFO [RS:0;a6f201fa5419:33465 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T15:23:49,115 INFO [RS:0;a6f201fa5419:33465 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T15:23:49,117 INFO [RS:0;a6f201fa5419:33465 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a6f201fa5419%2C33465%2C1731425027553, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/WALs/a6f201fa5419,33465,1731425027553, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/oldWALs, maxLogs=55 2024-11-12T15:23:49,139 INFO [RS:0;a6f201fa5419:33465 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/WALs/a6f201fa5419,33465,1731425027553/a6f201fa5419%2C33465%2C1731425027553.1731425029122 2024-11-12T15:23:49,139 DEBUG [RS:0;a6f201fa5419:33465 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T15:23:49,193 DEBUG [a6f201fa5419:36173 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T15:23:49,208 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a6f201fa5419,33465,1731425027553 2024-11-12T15:23:49,214 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a6f201fa5419,33465,1731425027553, state=OPENING 2024-11-12T15:23:49,301 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T15:23:49,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:49,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:49,315 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T15:23:49,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T15:23:49,318 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T15:23:49,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a6f201fa5419,33465,1731425027553}] 2024-11-12T15:23:49,495 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T15:23:49,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40547, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T15:23:49,505 INFO [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T15:23:49,505 INFO [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T15:23:49,506 INFO [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T15:23:49,507 INFO [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a6f201fa5419%2C33465%2C1731425027553.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/WALs/a6f201fa5419,33465,1731425027553, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/oldWALs, maxLogs=55 2024-11-12T15:23:49,526 INFO [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/WALs/a6f201fa5419,33465,1731425027553/a6f201fa5419%2C33465%2C1731425027553.meta.1731425029509.meta 2024-11-12T15:23:49,527 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T15:23:49,527 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:23:49,529 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T15:23:49,531 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T15:23:49,535 INFO [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T15:23:49,539 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T15:23:49,539 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:23:49,539 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T15:23:49,540 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T15:23:49,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T15:23:49,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T15:23:49,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:49,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:23:49,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T15:23:49,544 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T15:23:49,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:49,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:23:49,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T15:23:49,545 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T15:23:49,545 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:49,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:23:49,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T15:23:49,546 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T15:23:49,546 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:49,546 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:23:49,547 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T15:23:49,548 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740 2024-11-12T15:23:49,548 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740 2024-11-12T15:23:49,549 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T15:23:49,549 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T15:23:49,549 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T15:23:49,550 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T15:23:49,551 INFO [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59294036, jitterRate=-0.11645001173019409}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T15:23:49,551 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T15:23:49,552 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731425029540Writing region info on filesystem at 1731425029541 (+1 ms)Initializing all the Stores at 1731425029541Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731425029541Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731425029541Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731425029541Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731425029541Cleaning up temporary data from old regions at 1731425029549 (+8 ms)Running coprocessor post-open hooks at 1731425029551 (+2 ms)Region opened successfully at 1731425029552 (+1 ms) 2024-11-12T15:23:49,557 INFO [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731425029487 2024-11-12T15:23:49,568 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a6f201fa5419,33465,1731425027553 2024-11-12T15:23:49,569 DEBUG [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T15:23:49,569 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a6f201fa5419,33465,1731425027553, state=OPEN 2024-11-12T15:23:49,570 INFO [RS_OPEN_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T15:23:49,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T15:23:49,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T15:23:49,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T15:23:49,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T15:23:49,695 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a6f201fa5419,33465,1731425027553 2024-11-12T15:23:49,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T15:23:49,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a6f201fa5419,33465,1731425027553 in 373 msec 2024-11-12T15:23:49,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T15:23:49,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 925 msec 2024-11-12T15:23:49,706 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T15:23:49,706 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T15:23:49,724 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:23:49,725 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a6f201fa5419,33465,1731425027553, seqNum=-1] 2024-11-12T15:23:49,741 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:23:49,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40059, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:23:49,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1480 sec 2024-11-12T15:23:49,759 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731425029759, completionTime=-1 2024-11-12T15:23:49,762 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T15:23:49,762 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T15:23:49,786 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-12T15:23:49,786 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731425089786 2024-11-12T15:23:49,786 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731425149786 2024-11-12T15:23:49,786 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 24 msec 2024-11-12T15:23:49,789 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a6f201fa5419,36173,1731425026619-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:49,790 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a6f201fa5419,36173,1731425026619-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:49,790 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a6f201fa5419,36173,1731425026619-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:49,791 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a6f201fa5419:36173, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:49,792 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:49,793 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:49,799 DEBUG [master/a6f201fa5419:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T15:23:49,820 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.072sec 2024-11-12T15:23:49,821 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T15:23:49,822 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T15:23:49,823 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T15:23:49,824 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T15:23:49,824 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T15:23:49,825 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a6f201fa5419,36173,1731425026619-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T15:23:49,825 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a6f201fa5419,36173,1731425026619-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T15:23:49,829 DEBUG [master/a6f201fa5419:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T15:23:49,830 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T15:23:49,830 INFO [master/a6f201fa5419:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a6f201fa5419,36173,1731425026619-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:23:49,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1b595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T15:23:49,885 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T15:23:49,885 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T15:23:49,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a6f201fa5419,36173,-1 for getting cluster id 2024-11-12T15:23:49,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:23:49,897 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3493279-838d-4391-8325-94791f2fe6e7' 2024-11-12T15:23:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:23:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3493279-838d-4391-8325-94791f2fe6e7" 2024-11-12T15:23:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ba83a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T15:23:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a6f201fa5419,36173,-1] 2024-11-12T15:23:49,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:23:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:23:49,908 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:23:49,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5a12a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T15:23:49,911 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:23:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a6f201fa5419,33465,1731425027553, seqNum=-1] 2024-11-12T15:23:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:23:49,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:23:49,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a6f201fa5419,36173,1731425026619 2024-11-12T15:23:49,941 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T15:23:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a6f201fa5419,36173,1731425026619 2024-11-12T15:23:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@66e14f5b 2024-11-12T15:23:49,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T15:23:49,952 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T15:23:49,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:23:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-12T15:23:49,965 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T15:23:49,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-12T15:23:49,976 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:49,978 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T15:23:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T15:23:50,001 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => daa3014cd758d58a114efbea97cad0ae, NAME => 'table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85 2024-11-12T15:23:50,007 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:23:50,007 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing daa3014cd758d58a114efbea97cad0ae, disabling compactions & flushes 2024-11-12T15:23:50,007 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:50,007 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:50,007 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. after waiting 0 ms 2024-11-12T15:23:50,008 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:50,008 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:50,008 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for daa3014cd758d58a114efbea97cad0ae: Waiting for close lock at 1731425030007Disabling compacts and flushes for region at 1731425030007Disabling writes for close at 1731425030008 (+1 ms)Writing region close event to WAL at 1731425030008Closed at 1731425030008 2024-11-12T15:23:50,009 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T15:23:50,013 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1731425030009"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731425030009"}]},"ts":"1731425030009"} 2024-11-12T15:23:50,018 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T15:23:50,019 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T15:23:50,021 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731425030019"}]},"ts":"1731425030019"} 2024-11-12T15:23:50,025 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-12T15:23:50,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=daa3014cd758d58a114efbea97cad0ae, ASSIGN}] 2024-11-12T15:23:50,028 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=daa3014cd758d58a114efbea97cad0ae, ASSIGN 2024-11-12T15:23:50,029 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=daa3014cd758d58a114efbea97cad0ae, ASSIGN; state=OFFLINE, location=a6f201fa5419,33465,1731425027553; forceNewPlan=false, retain=false 2024-11-12T15:23:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T15:23:50,181 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=daa3014cd758d58a114efbea97cad0ae, regionState=OPENING, regionLocation=a6f201fa5419,33465,1731425027553 2024-11-12T15:23:50,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=daa3014cd758d58a114efbea97cad0ae, ASSIGN because future has completed 2024-11-12T15:23:50,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure daa3014cd758d58a114efbea97cad0ae, server=a6f201fa5419,33465,1731425027553}] 2024-11-12T15:23:50,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T15:23:50,348 INFO [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:50,348 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => daa3014cd758d58a114efbea97cad0ae, NAME => 'table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:23:50,350 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:50,350 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:23:50,350 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:50,350 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:50,352 INFO [StoreOpener-daa3014cd758d58a114efbea97cad0ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:50,357 INFO [StoreOpener-daa3014cd758d58a114efbea97cad0ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region daa3014cd758d58a114efbea97cad0ae columnFamilyName family 2024-11-12T15:23:50,357 DEBUG [StoreOpener-daa3014cd758d58a114efbea97cad0ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:23:50,357 INFO [StoreOpener-daa3014cd758d58a114efbea97cad0ae-1 {}] regionserver.HStore(327): Store=daa3014cd758d58a114efbea97cad0ae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:23:50,358 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:50,360 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:50,360 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:50,361 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:50,361 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:50,363 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:50,369 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T15:23:50,370 INFO [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened daa3014cd758d58a114efbea97cad0ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60440257, jitterRate=-0.09936998784542084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T15:23:50,370 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:50,371 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for daa3014cd758d58a114efbea97cad0ae: Running coprocessor pre-open hook at 1731425030351Writing region info on filesystem at 1731425030351Initializing all the Stores at 1731425030351Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731425030352 (+1 ms)Cleaning up temporary data from old regions at 1731425030361 (+9 ms)Running coprocessor post-open hooks at 1731425030370 (+9 ms)Region opened successfully at 1731425030371 (+1 ms) 2024-11-12T15:23:50,373 INFO [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae., pid=6, masterSystemTime=1731425030343 2024-11-12T15:23:50,375 DEBUG [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:50,376 INFO [RS_OPEN_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:50,376 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=daa3014cd758d58a114efbea97cad0ae, regionState=OPEN, openSeqNum=2, regionLocation=a6f201fa5419,33465,1731425027553 2024-11-12T15:23:50,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure daa3014cd758d58a114efbea97cad0ae, server=a6f201fa5419,33465,1731425027553 because future has completed 2024-11-12T15:23:50,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T15:23:50,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure daa3014cd758d58a114efbea97cad0ae, server=a6f201fa5419,33465,1731425027553 in 193 msec 2024-11-12T15:23:50,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T15:23:50,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=daa3014cd758d58a114efbea97cad0ae, ASSIGN in 356 msec 2024-11-12T15:23:50,386 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T15:23:50,386 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731425030386"}]},"ts":"1731425030386"} 2024-11-12T15:23:50,389 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-12T15:23:50,390 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T15:23:50,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 431 msec 2024-11-12T15:23:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T15:23:50,619 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-12T15:23:50,619 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-12T15:23:50,623 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T15:23:50,632 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-12T15:23:50,633 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T15:23:50,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-12T15:23:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae., hostname=a6f201fa5419,33465,1731425027553, seqNum=2] 2024-11-12T15:23:50,653 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing daa3014cd758d58a114efbea97cad0ae 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-12T15:23:50,674 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f7956ceca9244e1eb18c96c2c9dad60f 2024-11-12T15:23:50,707 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/.tmp/family/f7956ceca9244e1eb18c96c2c9dad60f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-12T15:23:50,711 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/.tmp/family/f7956ceca9244e1eb18c96c2c9dad60f 2024-11-12T15:23:50,742 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/.tmp/family/f7956ceca9244e1eb18c96c2c9dad60f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/family/f7956ceca9244e1eb18c96c2c9dad60f 2024-11-12T15:23:50,743 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/family/f7956ceca9244e1eb18c96c2c9dad60f, delay=982 ms 2024-11-12T15:23:50,745 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/family/f7956ceca9244e1eb18c96c2c9dad60f, entries=2, sequenceid=6, filesize=4.9 K 2024-11-12T15:23:50,750 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for daa3014cd758d58a114efbea97cad0ae in 97ms, sequenceid=6, compaction requested=false 2024-11-12T15:23:50,751 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-12T15:23:50,753 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for daa3014cd758d58a114efbea97cad0ae: 2024-11-12T15:23:51,728 DEBUG [hfile-prefetch-1731425030743 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f7956ceca9244e1eb18c96c2c9dad60f_0. 2024-11-12T15:23:51,733 DEBUG [hfile-prefetch-1731425030743 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f7956ceca9244e1eb18c96c2c9dad60f_117. 2024-11-12T15:23:51,734 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/family/f7956ceca9244e1eb18c96c2c9dad60f, with total blocks 2, and data blocks 1 2024-11-12T15:23:51,734 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-12T15:23:51,735 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/family/f7956ceca9244e1eb18c96c2c9dad60f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-12T15:23:51,735 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/family/f7956ceca9244e1eb18c96c2c9dad60f. Will sleep 100ms and try the verification again. 2024-11-12T15:23:51,836 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/family/f7956ceca9244e1eb18c96c2c9dad60f, with total blocks 2, and data blocks 1 2024-11-12T15:23:51,836 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-12T15:23:51,836 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/family/f7956ceca9244e1eb18c96c2c9dad60f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-12T15:23:51,836 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/family/f7956ceca9244e1eb18c96c2c9dad60f. Will sleep 100ms and try the verification again. 2024-11-12T15:23:51,937 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/family/f7956ceca9244e1eb18c96c2c9dad60f, with total blocks 2, and data blocks 1 2024-11-12T15:23:51,937 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-12T15:23:51,937 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2294): found block for file f7956ceca9244e1eb18c96c2c9dad60f in the backing map. Acquiring read lock for offset 0 2024-11-12T15:23:51,937 DEBUG [hfile-prefetch-1731425030743 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/family/f7956ceca9244e1eb18c96c2c9dad60f has now been fully cached. 2024-11-12T15:23:51,938 DEBUG [hfile-prefetch-1731425030743 {}] hfile.PrefetchExecutor(121): Prefetch completed for f7956ceca9244e1eb18c96c2c9dad60f 2024-11-12T15:23:52,638 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T15:23:52,639 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T15:23:52,640 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T15:23:52,640 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-12T15:23:52,662 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-12T15:23:52,662 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731425032639 2024-11-12T15:23:52,662 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T15:23:52,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T15:23:52,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T15:23:52,764 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;a6f201fa5419:33465,5,FailOnTimeoutGroup] 2024-11-12T15:23:52,764 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a6f201fa5419,33465,1731425027553' ***** 2024-11-12T15:23:52,764 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-12T15:23:52,764 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-12T15:23:52,765 INFO [RS:0;a6f201fa5419:33465 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T15:23:52,765 INFO [RS:0;a6f201fa5419:33465 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/bucket.persistence 2024-11-12T15:23:52,765 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T15:23:52,765 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T15:23:52,765 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T15:23:52,765 INFO [RS:0;a6f201fa5419:33465 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T15:23:52,765 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T15:23:52,775 DEBUG [RS:0;a6f201fa5419:33465 {}] bucket.BucketCache(1488): Thread RS:0;a6f201fa5419:33465 started persisting bucket cache to file 2024-11-12T15:23:52,776 DEBUG [RS:0;a6f201fa5419:33465 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T15:23:52,776 DEBUG [RS:0;a6f201fa5419:33465 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-12T15:23:52,779 DEBUG [RS:0;a6f201fa5419:33465 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-12T15:23:52,779 DEBUG [RS:0;a6f201fa5419:33465 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731425032775 2024-11-12T15:23:52,779 DEBUG [RS:0;a6f201fa5419:33465 {}] bucket.BucketCache(1510): Thread RS:0;a6f201fa5419:33465 finished persisting bucket cache to file, renaming 2024-11-12T15:23:52,838 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T15:23:52,838 INFO [RS:0;a6f201fa5419:33465 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T15:23:52,838 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T15:23:52,839 INFO [RS:0;a6f201fa5419:33465 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T15:23:52,839 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(3091): Received CLOSE for daa3014cd758d58a114efbea97cad0ae 2024-11-12T15:23:52,840 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(959): stopping server a6f201fa5419,33465,1731425027553 2024-11-12T15:23:52,841 INFO [RS:0;a6f201fa5419:33465 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T15:23:52,841 INFO [RS_CLOSE_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T15:23:52,842 INFO [RS:0;a6f201fa5419:33465 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a6f201fa5419:33465. 2024-11-12T15:23:52,842 DEBUG [RS_CLOSE_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing daa3014cd758d58a114efbea97cad0ae, disabling compactions & flushes 2024-11-12T15:23:52,842 DEBUG [RS:0;a6f201fa5419:33465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:23:52,842 INFO [RS_CLOSE_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:52,842 DEBUG [RS_CLOSE_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:52,843 DEBUG [RS_CLOSE_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. after waiting 0 ms 2024-11-12T15:23:52,843 DEBUG [RS_CLOSE_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:52,854 DEBUG [RS_CLOSE_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/default/table1/daa3014cd758d58a114efbea97cad0ae/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-12T15:23:52,855 INFO [RS_CLOSE_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:52,855 DEBUG [RS_CLOSE_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for daa3014cd758d58a114efbea97cad0ae: Waiting for close lock at 1731425032841Running coprocessor pre-close hooks at 1731425032842 (+1 ms)Disabling compacts and flushes for region at 1731425032842Disabling writes for close at 1731425032843 (+1 ms)Writing region close event to WAL at 1731425032846 (+3 ms)Running coprocessor post-close hooks at 1731425032854 (+8 ms)Closed at 1731425032855 (+1 ms) 2024-11-12T15:23:52,855 DEBUG [RS:0;a6f201fa5419:33465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:23:52,855 DEBUG [RS_CLOSE_REGION-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae. 2024-11-12T15:23:52,856 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T15:23:52,856 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T15:23:52,856 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T15:23:52,856 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T15:23:52,857 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T15:23:52,857 INFO [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T15:23:52,857 DEBUG [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T15:23:52,857 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T15:23:52,857 INFO [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T15:23:52,857 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T15:23:52,857 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T15:23:52,857 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T15:23:52,857 DEBUG [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T15:23:52,857 INFO [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-12T15:23:52,863 INFO [regionserver/a6f201fa5419:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T15:23:52,866 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7c8c176800e54a16bf39b2f6c8a554d8 2024-11-12T15:23:52,881 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/.tmp/info/7c8c176800e54a16bf39b2f6c8a554d8 is 133, key is table1,,1731425029954.daa3014cd758d58a114efbea97cad0ae./info:regioninfo/1731425030376/Put/seqid=0 2024-11-12T15:23:52,881 INFO [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/.tmp/info/7c8c176800e54a16bf39b2f6c8a554d8 2024-11-12T15:23:52,889 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4247a78dec954af3821bc61588c001be 2024-11-12T15:23:52,901 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/.tmp/ns/4247a78dec954af3821bc61588c001be is 43, key is default/ns:d/1731425029747/Put/seqid=0 2024-11-12T15:23:52,902 INFO [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/.tmp/ns/4247a78dec954af3821bc61588c001be 2024-11-12T15:23:52,911 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 73cc2566534a4f7da9bdd0585a82282e 2024-11-12T15:23:52,924 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/.tmp/table/73cc2566534a4f7da9bdd0585a82282e is 42, key is table1/table:state/1731425030386/Put/seqid=0 2024-11-12T15:23:52,924 INFO [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/.tmp/table/73cc2566534a4f7da9bdd0585a82282e 2024-11-12T15:23:52,927 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/.tmp/info/7c8c176800e54a16bf39b2f6c8a554d8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/info/7c8c176800e54a16bf39b2f6c8a554d8 2024-11-12T15:23:52,928 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/info/7c8c176800e54a16bf39b2f6c8a554d8, delay=938 ms 2024-11-12T15:23:52,930 INFO [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/info/7c8c176800e54a16bf39b2f6c8a554d8, entries=10, sequenceid=11, filesize=6.3 K 2024-11-12T15:23:52,930 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/.tmp/ns/4247a78dec954af3821bc61588c001be as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/ns/4247a78dec954af3821bc61588c001be 2024-11-12T15:23:52,932 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/ns/4247a78dec954af3821bc61588c001be, delay=995 ms 2024-11-12T15:23:52,933 INFO [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/ns/4247a78dec954af3821bc61588c001be, entries=2, sequenceid=11, filesize=5.0 K 2024-11-12T15:23:52,934 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/.tmp/table/73cc2566534a4f7da9bdd0585a82282e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/table/73cc2566534a4f7da9bdd0585a82282e 2024-11-12T15:23:52,935 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/table/73cc2566534a4f7da9bdd0585a82282e, delay=984 ms 2024-11-12T15:23:52,936 INFO [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/table/73cc2566534a4f7da9bdd0585a82282e, entries=2, sequenceid=11, filesize=5.1 K 2024-11-12T15:23:52,937 INFO [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=11, compaction requested=false 2024-11-12T15:23:52,937 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T15:23:52,939 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/info/7c8c176800e54a16bf39b2f6c8a554d8 2024-11-12T15:23:52,939 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/ns/4247a78dec954af3821bc61588c001be 2024-11-12T15:23:52,940 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/table/73cc2566534a4f7da9bdd0585a82282e 2024-11-12T15:23:52,944 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-12T15:23:52,944 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T15:23:52,944 INFO [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T15:23:52,945 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731425032857Running coprocessor pre-close hooks at 1731425032857Disabling compacts and flushes for region at 1731425032857Disabling writes for close at 1731425032857Obtaining lock to block concurrent updates at 1731425032857Preparing flush snapshotting stores in 1588230740 at 1731425032857Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1731425032858 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731425032858Flushing 1588230740/info: creating writer at 1731425032859 (+1 ms)Flushing 1588230740/info: appending metadata at 1731425032880 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731425032880Flushing 1588230740/ns: creating writer at 1731425032884 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731425032900 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731425032900Flushing 1588230740/table: creating writer at 1731425032905 (+5 ms)Flushing 1588230740/table: appending metadata at 1731425032923 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731425032923Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70ff6894: reopening flushed file at 1731425032927 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@590ca42b: reopening flushed file at 1731425032930 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63b1ad24: reopening flushed file at 1731425032933 (+3 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=11, compaction requested=false at 1731425032937 (+4 ms)Writing region close event to WAL at 1731425032940 (+3 ms)Running coprocessor post-close hooks at 1731425032944 (+4 ms)Closed at 1731425032944 2024-11-12T15:23:52,945 DEBUG [RS_CLOSE_META-regionserver/a6f201fa5419:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T15:23:52,969 INFO [regionserver/a6f201fa5419:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T15:23:52,969 INFO [regionserver/a6f201fa5419:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T15:23:53,058 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(976): stopping server a6f201fa5419,33465,1731425027553; all regions closed. 2024-11-12T15:23:53,067 DEBUG [RS:0;a6f201fa5419:33465 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/oldWALs 2024-11-12T15:23:53,067 INFO [RS:0;a6f201fa5419:33465 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a6f201fa5419%2C33465%2C1731425027553.meta:.meta(num 1731425029509) 2024-11-12T15:23:53,070 DEBUG [RS:0;a6f201fa5419:33465 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/oldWALs 2024-11-12T15:23:53,070 INFO [RS:0;a6f201fa5419:33465 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a6f201fa5419%2C33465%2C1731425027553:(num 1731425029122) 2024-11-12T15:23:53,070 DEBUG [RS:0;a6f201fa5419:33465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:23:53,070 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T15:23:53,070 INFO [RS:0;a6f201fa5419:33465 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T15:23:53,070 INFO [RS:0;a6f201fa5419:33465 {}] hbase.ChoreService(370): Chore service for: regionserver/a6f201fa5419:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-12T15:23:53,070 INFO [RS:0;a6f201fa5419:33465 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T15:23:53,070 INFO [regionserver/a6f201fa5419:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T15:23:53,071 INFO [RS:0;a6f201fa5419:33465 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33465 2024-11-12T15:23:53,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T15:23:53,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a6f201fa5419,33465,1731425027553 2024-11-12T15:23:53,093 INFO [RS:0;a6f201fa5419:33465 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T15:23:53,104 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a6f201fa5419,33465,1731425027553] 2024-11-12T15:23:53,114 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a6f201fa5419,33465,1731425027553 already deleted, retry=false 2024-11-12T15:23:53,114 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a6f201fa5419,33465,1731425027553 on a6f201fa5419,36173,1731425026619 2024-11-12T15:23:53,122 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a6f201fa5419,33465,1731425027553, splitWal=true, meta=true 2024-11-12T15:23:53,124 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a6f201fa5419,33465,1731425027553, splitWal=true, meta=true 2024-11-12T15:23:53,125 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=7 for a6f201fa5419,33465,1731425027553 (carryingMeta=true) a6f201fa5419,33465,1731425027553/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@245758fd[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-12T15:23:53,125 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure a6f201fa5419,33465,1731425027553, splitWal=true, meta=true, isMeta: true 2024-11-12T15:23:53,126 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/WALs/a6f201fa5419,33465,1731425027553-splitting 2024-11-12T15:23:53,126 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/WALs/a6f201fa5419,33465,1731425027553-splitting dir is empty, no logs to split. 2024-11-12T15:23:53,126 INFO [PEWorker-2 {}] master.SplitWALManager(105): a6f201fa5419,33465,1731425027553 WAL count=0, meta=true 2024-11-12T15:23:53,128 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/WALs/a6f201fa5419,33465,1731425027553-splitting dir is empty, no logs to split. 2024-11-12T15:23:53,128 INFO [PEWorker-2 {}] master.SplitWALManager(105): a6f201fa5419,33465,1731425027553 WAL count=0, meta=true 2024-11-12T15:23:53,128 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if a6f201fa5419,33465,1731425027553 WAL splitting is done? wals=0, meta=true 2024-11-12T15:23:53,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T15:23:53,130 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T15:23:53,131 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T15:23:53,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T15:23:53,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33465-0x1012fa53cbf0001, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T15:23:53,205 INFO [RS:0;a6f201fa5419:33465 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T15:23:53,205 INFO [RS:0;a6f201fa5419:33465 {}] regionserver.HRegionServer(1031): Exiting; stopping=a6f201fa5419,33465,1731425027553; zookeeper connection closed. 2024-11-12T15:23:53,206 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@13208dc2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@13208dc2 2024-11-12T15:23:53,281 WARN [a6f201fa5419:36173 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-12T15:23:53,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T15:23:53,765 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T15:23:53,765 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:23:53,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:23:53,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:23:53,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:23:53,767 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T15:23:53,767 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1590087585, stopped=false 2024-11-12T15:23:53,767 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a6f201fa5419,36173,1731425026619 2024-11-12T15:23:53,767 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'a6f201fa5419,36173,1731425026619' ***** 2024-11-12T15:23:53,767 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-12T15:23:53,767 INFO [M:0;a6f201fa5419:36173 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T15:23:53,768 INFO [M:0;a6f201fa5419:36173 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;a6f201fa5419:36173. 2024-11-12T15:23:53,768 DEBUG [M:0;a6f201fa5419:36173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:23:53,768 DEBUG [M:0;a6f201fa5419:36173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:23:53,768 INFO [M:0;a6f201fa5419:36173 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T15:23:53,768 DEBUG [M:0;a6f201fa5419:36173 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T15:23:53,768 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T15:23:53,768 DEBUG [M:0;a6f201fa5419:36173 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T15:23:53,768 DEBUG [master/a6f201fa5419:0:becomeActiveMaster-HFileCleaner.large.0-1731425028711 {}] cleaner.HFileCleaner(306): Exit Thread[master/a6f201fa5419:0:becomeActiveMaster-HFileCleaner.large.0-1731425028711,5,FailOnTimeoutGroup] 2024-11-12T15:23:53,769 DEBUG [master/a6f201fa5419:0:becomeActiveMaster-HFileCleaner.small.0-1731425028712 {}] cleaner.HFileCleaner(306): Exit Thread[master/a6f201fa5419:0:becomeActiveMaster-HFileCleaner.small.0-1731425028712,5,FailOnTimeoutGroup] 2024-11-12T15:23:53,769 INFO [M:0;a6f201fa5419:36173 {}] hbase.ChoreService(370): Chore service for: master/a6f201fa5419:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T15:23:53,769 INFO [M:0;a6f201fa5419:36173 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T15:23:53,769 DEBUG [M:0;a6f201fa5419:36173 {}] master.HMaster(1795): Stopping service threads 2024-11-12T15:23:53,769 INFO [M:0;a6f201fa5419:36173 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T15:23:53,769 INFO [M:0;a6f201fa5419:36173 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T15:23:53,770 INFO [M:0;a6f201fa5419:36173 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T15:23:53,771 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T15:23:53,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T15:23:53,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T15:23:53,880 DEBUG [M:0;a6f201fa5419:36173 {}] zookeeper.ZKUtil(347): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T15:23:53,880 WARN [M:0;a6f201fa5419:36173 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T15:23:53,881 INFO [M:0;a6f201fa5419:36173 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/.lastflushedseqids 2024-11-12T15:23:53,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T15:23:53,893 INFO [M:0;a6f201fa5419:36173 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T15:23:53,894 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-12T15:23:53,894 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T15:23:54,032 DEBUG [a6f201fa5419:36173 {}] assignment.AssignmentManager(2447): Stopped! Dropping assign of 1 queued regions. 2024-11-12T15:23:54,032 INFO [M:0;a6f201fa5419:36173 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T15:23:54,033 DEBUG [M:0;a6f201fa5419:36173 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T15:23:54,033 INFO [M:0;a6f201fa5419:36173 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:23:54,033 DEBUG [M:0;a6f201fa5419:36173 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:23:54,033 DEBUG [M:0;a6f201fa5419:36173 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T15:23:54,033 DEBUG [M:0;a6f201fa5419:36173 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:23:54,033 INFO [M:0;a6f201fa5419:36173 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.17 KB heapSize=30.80 KB 2024-11-12T15:23:54,045 DEBUG [M:0;a6f201fa5419:36173 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 23925b507ef54be5b7aad360c982a228 2024-11-12T15:23:54,061 DEBUG [M:0;a6f201fa5419:36173 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/23925b507ef54be5b7aad360c982a228 is 82, key is hbase:meta,,1/info:regioninfo/1731425029567/Put/seqid=0 2024-11-12T15:23:54,063 INFO [M:0;a6f201fa5419:36173 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/23925b507ef54be5b7aad360c982a228 2024-11-12T15:23:54,074 DEBUG [M:0;a6f201fa5419:36173 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d7206865b4e44340b6bc679a7db794c9 2024-11-12T15:23:54,089 DEBUG [M:0;a6f201fa5419:36173 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d7206865b4e44340b6bc679a7db794c9 is 706, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731425030391/Put/seqid=0 2024-11-12T15:23:54,090 INFO [M:0;a6f201fa5419:36173 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.56 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d7206865b4e44340b6bc679a7db794c9 2024-11-12T15:23:54,099 DEBUG [M:0;a6f201fa5419:36173 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7976418214f64280878ec75cb6f3c98b 2024-11-12T15:23:54,113 DEBUG [M:0;a6f201fa5419:36173 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7976418214f64280878ec75cb6f3c98b is 58, key is a6f201fa5419,33465,1731425027553/rs:/1731425033126/DeleteFamily/seqid=0 2024-11-12T15:23:54,114 INFO [M:0;a6f201fa5419:36173 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7976418214f64280878ec75cb6f3c98b 2024-11-12T15:23:54,117 INFO [M:0;a6f201fa5419:36173 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7976418214f64280878ec75cb6f3c98b 2024-11-12T15:23:54,118 DEBUG [M:0;a6f201fa5419:36173 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/23925b507ef54be5b7aad360c982a228 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/23925b507ef54be5b7aad360c982a228 2024-11-12T15:23:54,119 INFO [M:0;a6f201fa5419:36173 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/23925b507ef54be5b7aad360c982a228, entries=8, sequenceid=64, filesize=5.5 K 2024-11-12T15:23:54,120 DEBUG [M:0;a6f201fa5419:36173 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d7206865b4e44340b6bc679a7db794c9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d7206865b4e44340b6bc679a7db794c9 2024-11-12T15:23:54,122 INFO [M:0;a6f201fa5419:36173 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d7206865b4e44340b6bc679a7db794c9, entries=8, sequenceid=64, filesize=6.5 K 2024-11-12T15:23:54,122 DEBUG [M:0;a6f201fa5419:36173 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7976418214f64280878ec75cb6f3c98b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7976418214f64280878ec75cb6f3c98b 2024-11-12T15:23:54,125 INFO [M:0;a6f201fa5419:36173 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7976418214f64280878ec75cb6f3c98b 2024-11-12T15:23:54,125 INFO [M:0;a6f201fa5419:36173 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/5896ee03-0e8a-9165-8c61-689828de3b71/458171c8-604f-9708-ea0f-1d939d672e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7976418214f64280878ec75cb6f3c98b, entries=1, sequenceid=64, filesize=5.2 K 2024-11-12T15:23:54,126 INFO [M:0;a6f201fa5419:36173 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=64, compaction requested=false 2024-11-12T15:23:54,134 INFO [M:0;a6f201fa5419:36173 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:23:54,134 DEBUG [M:0;a6f201fa5419:36173 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731425034033Disabling compacts and flushes for region at 1731425034033Disabling writes for close at 1731425034033Obtaining lock to block concurrent updates at 1731425034033Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731425034033Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24747, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1731425034034 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731425034034Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731425034034Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731425034061 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731425034061Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731425034067 (+6 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731425034088 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731425034088Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731425034092 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731425034112 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731425034112Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43d87551: reopening flushed file at 1731425034117 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d6fc3ba: reopening flushed file at 1731425034120 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@295d2e07: reopening flushed file at 1731425034122 (+2 ms)Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=64, compaction requested=false at 1731425034127 (+5 ms)Writing region close event to WAL at 1731425034134 (+7 ms)Closed at 1731425034134 2024-11-12T15:23:54,140 INFO [M:0;a6f201fa5419:36173 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T15:23:54,140 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T15:23:54,140 INFO [M:0;a6f201fa5419:36173 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36173 2024-11-12T15:23:54,140 INFO [M:0;a6f201fa5419:36173 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T15:23:54,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T15:23:54,254 INFO [M:0;a6f201fa5419:36173 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T15:23:54,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012fa53cbf0000, quorum=127.0.0.1:51493, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T15:23:54,259 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T15:23:54,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-12T15:23:54,280 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T15:23:54,288 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=45 (was 12) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a6f201fa5419:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731425032928 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731425032935 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731425030743 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/a6f201fa5419:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731425032932 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a6f201fa5419:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=327 (was 342), ProcessCount=11 (was 11), AvailableMemoryMB=9457 (was 9665)