2024-11-24 08:54:46,067 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 08:54:46,084 main DEBUG Took 0.014644 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 08:54:46,084 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 08:54:46,085 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 08:54:46,086 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 08:54:46,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,094 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 08:54:46,105 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,106 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,107 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,108 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,108 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,109 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,110 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,110 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,111 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,112 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,112 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,113 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,113 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,113 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,114 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,114 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,115 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,115 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,115 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,116 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,116 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,117 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,117 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:54:46,117 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,118 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 08:54:46,119 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:54:46,120 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 08:54:46,122 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 08:54:46,123 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 08:54:46,124 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 08:54:46,124 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 08:54:46,133 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 08:54:46,135 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 08:54:46,137 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 08:54:46,138 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 08:54:46,138 main DEBUG createAppenders(={Console}) 2024-11-24 08:54:46,139 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-24 08:54:46,139 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 08:54:46,139 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-24 08:54:46,140 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 08:54:46,140 main DEBUG OutputStream closed 2024-11-24 08:54:46,140 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 08:54:46,141 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 08:54:46,141 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-24 08:54:46,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 08:54:46,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 08:54:46,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 08:54:46,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 08:54:46,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 08:54:46,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 08:54:46,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 08:54:46,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 08:54:46,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 08:54:46,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 08:54:46,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 08:54:46,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 08:54:46,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 08:54:46,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 08:54:46,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 08:54:46,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 08:54:46,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 08:54:46,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 08:54:46,222 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 08:54:46,223 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-24 08:54:46,223 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 08:54:46,224 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-24T08:54:46,236 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-24 08:54:46,239 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 08:54:46,239 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T08:54:46,476 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7 2024-11-24T08:54:46,498 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=111, ProcessCount=11, AvailableMemoryMB=2569 2024-11-24T08:54:46,580 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T08:54:46,662 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:46,675 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/cluster_de6e566c-32aa-81e0-83f1-780b3e270247, deleteOnExit=true 2024-11-24T08:54:46,779 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/cluster_de6e566c-32aa-81e0-83f1-780b3e270247/zookeeper_0, clientPort=64764, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/cluster_de6e566c-32aa-81e0-83f1-780b3e270247/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/cluster_de6e566c-32aa-81e0-83f1-780b3e270247/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T08:54:46,795 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64764 2024-11-24T08:54:46,795 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:46,906 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed with version=8 2024-11-24T08:54:46,907 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/hbase-staging 2024-11-24T08:54:47,008 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T08:54:47,228 INFO [Time-limited test {}] client.ConnectionUtils(128): master/469387a2cdb6:0 server-side Connection retries=45 2024-11-24T08:54:47,237 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:47,238 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:47,247 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:54:47,247 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:47,247 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:54:47,373 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T08:54:47,448 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T08:54:47,491 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T08:54:47,501 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T08:54:47,504 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:54:47,540 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 90745 (auto-detected) 2024-11-24T08:54:47,542 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T08:54:47,563 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33061 2024-11-24T08:54:47,592 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33061 connecting to ZooKeeper ensemble=127.0.0.1:64764 2024-11-24T08:54:47,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:330610x0, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:54:47,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33061-0x10070f0cb510000 connected 2024-11-24T08:54:47,655 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:47,656 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:47,663 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:47,667 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed, hbase.cluster.distributed=false 2024-11-24T08:54:47,689 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:54:47,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33061 2024-11-24T08:54:47,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33061 2024-11-24T08:54:47,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33061 2024-11-24T08:54:47,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33061 2024-11-24T08:54:47,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33061 2024-11-24T08:54:47,775 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/469387a2cdb6:0 server-side Connection retries=45 2024-11-24T08:54:47,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:47,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:47,777 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:54:47,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:47,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:54:47,781 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T08:54:47,784 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:54:47,785 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34649 2024-11-24T08:54:47,787 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34649 connecting to ZooKeeper ensemble=127.0.0.1:64764 2024-11-24T08:54:47,787 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:47,788 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:47,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:346490x0, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:54:47,792 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:346490x0, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:47,792 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34649-0x10070f0cb510001 connected 2024-11-24T08:54:47,797 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T08:54:47,811 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/bucket.cache 2024-11-24T08:54:47,819 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T08:54:47,821 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T08:54:47,823 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T08:54:47,825 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T08:54:47,825 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T08:54:47,830 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T08:54:47,836 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T08:54:47,836 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T08:54:47,836 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/bucket.persistence is complete. 2024-11-24T08:54:47,839 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T08:54:47,844 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:54:47,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34649 2024-11-24T08:54:47,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34649 2024-11-24T08:54:47,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34649 2024-11-24T08:54:47,855 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34649 2024-11-24T08:54:47,857 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34649 2024-11-24T08:54:47,883 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/469387a2cdb6:0 server-side Connection retries=45 2024-11-24T08:54:47,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:47,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:47,884 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:54:47,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:47,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:54:47,885 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T08:54:47,885 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:54:47,886 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32991 2024-11-24T08:54:47,889 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32991 connecting to ZooKeeper ensemble=127.0.0.1:64764 2024-11-24T08:54:47,890 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:47,891 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:47,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:329910x0, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:54:47,896 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:329910x0, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:47,896 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32991-0x10070f0cb510002 connected 2024-11-24T08:54:47,896 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T08:54:47,897 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/bucket.cache 2024-11-24T08:54:47,897 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T08:54:47,898 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T08:54:47,898 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T08:54:47,901 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T08:54:47,901 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T08:54:47,902 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T08:54:47,903 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/bucket.persistence is complete. 2024-11-24T08:54:47,904 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T08:54:47,905 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T08:54:47,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T08:54:47,909 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:54:47,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32991 2024-11-24T08:54:47,913 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32991 2024-11-24T08:54:47,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32991 2024-11-24T08:54:47,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32991 2024-11-24T08:54:47,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32991 2024-11-24T08:54:47,934 DEBUG [M:0;469387a2cdb6:33061 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;469387a2cdb6:33061 2024-11-24T08:54:47,937 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/469387a2cdb6,33061,1732438487075 2024-11-24T08:54:47,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:47,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:47,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:47,946 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/469387a2cdb6,33061,1732438487075 2024-11-24T08:54:47,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T08:54:47,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T08:54:47,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:47,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:47,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:47,977 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T08:54:47,978 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/469387a2cdb6,33061,1732438487075 from backup master directory 2024-11-24T08:54:47,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:47,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:47,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/469387a2cdb6,33061,1732438487075 2024-11-24T08:54:47,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:47,983 WARN [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:54:47,983 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=469387a2cdb6,33061,1732438487075 2024-11-24T08:54:47,985 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T08:54:47,987 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T08:54:47,995 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/hbase.id] with ID: 13f39dd5-6acd-4f1f-9c5d-94e01df1863e 2024-11-24T08:54:47,995 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/.tmp/hbase.id 2024-11-24T08:54:48,001 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/hbase.id] 2024-11-24T08:54:48,041 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:48,047 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T08:54:48,067 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-24T08:54:48,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,097 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:54:48,101 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T08:54:48,141 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:54:48,146 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:54:48,198 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store 2024-11-24T08:54:48,219 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T08:54:48,223 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:48,225 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T08:54:48,225 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:48,225 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:48,227 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T08:54:48,228 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:48,228 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:48,229 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732438488225Disabling compacts and flushes for region at 1732438488225Disabling writes for close at 1732438488228 (+3 ms)Writing region close event to WAL at 1732438488228Closed at 1732438488228 2024-11-24T08:54:48,230 WARN [master/469387a2cdb6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/.initializing 2024-11-24T08:54:48,230 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/WALs/469387a2cdb6,33061,1732438487075 2024-11-24T08:54:48,242 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:54:48,255 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=469387a2cdb6%2C33061%2C1732438487075, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/WALs/469387a2cdb6,33061,1732438487075, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/oldWALs, maxLogs=10 2024-11-24T08:54:48,291 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/WALs/469387a2cdb6,33061,1732438487075/469387a2cdb6%2C33061%2C1732438487075.1732438488259 2024-11-24T08:54:48,292 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T08:54:48,293 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:54:48,294 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:48,297 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,298 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,334 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,361 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T08:54:48,365 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:48,366 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:48,367 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T08:54:48,372 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:48,373 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:54:48,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,378 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T08:54:48,378 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:48,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:54:48,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,384 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T08:54:48,384 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:48,385 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:54:48,385 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,389 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,389 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,392 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,393 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,396 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T08:54:48,398 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:48,405 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:54:48,407 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73324378, jitterRate=0.09261837601661682}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T08:54:48,418 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732438488311Initializing all the Stores at 1732438488311Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438488312 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438488312Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438488313 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438488313Cleaning up temporary data from old regions at 1732438488393 (+80 ms)Region opened successfully at 1732438488418 (+25 ms) 2024-11-24T08:54:48,420 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T08:54:48,459 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3288fea9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=469387a2cdb6/172.17.0.2:0 2024-11-24T08:54:48,500 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T08:54:48,513 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T08:54:48,513 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T08:54:48,517 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T08:54:48,517 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T08:54:48,524 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-24T08:54:48,524 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T08:54:48,552 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T08:54:48,562 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T08:54:48,565 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T08:54:48,568 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T08:54:48,570 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T08:54:48,571 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T08:54:48,574 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T08:54:48,578 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T08:54:48,579 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T08:54:48,581 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T08:54:48,582 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T08:54:48,603 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T08:54:48,604 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T08:54:48,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:48,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:48,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:48,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,621 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=469387a2cdb6,33061,1732438487075, sessionid=0x10070f0cb510000, setting cluster-up flag (Was=false) 2024-11-24T08:54:48,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,638 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T08:54:48,640 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=469387a2cdb6,33061,1732438487075 2024-11-24T08:54:48,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:48,648 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T08:54:48,650 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=469387a2cdb6,33061,1732438487075 2024-11-24T08:54:48,656 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T08:54:48,722 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(746): ClusterId : 13f39dd5-6acd-4f1f-9c5d-94e01df1863e 2024-11-24T08:54:48,722 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer(746): ClusterId : 13f39dd5-6acd-4f1f-9c5d-94e01df1863e 2024-11-24T08:54:48,725 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T08:54:48,725 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T08:54:48,728 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T08:54:48,730 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T08:54:48,730 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T08:54:48,730 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T08:54:48,730 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T08:54:48,733 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T08:54:48,733 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T08:54:48,734 DEBUG [RS:0;469387a2cdb6:34649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58379497, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=469387a2cdb6/172.17.0.2:0 2024-11-24T08:54:48,734 DEBUG [RS:1;469387a2cdb6:32991 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d1e20df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=469387a2cdb6/172.17.0.2:0 2024-11-24T08:54:48,743 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T08:54:48,750 DEBUG [RS:0;469387a2cdb6:34649 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;469387a2cdb6:34649 2024-11-24T08:54:48,752 DEBUG [RS:1;469387a2cdb6:32991 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;469387a2cdb6:32991 2024-11-24T08:54:48,753 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T08:54:48,754 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T08:54:48,754 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T08:54:48,754 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T08:54:48,754 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T08:54:48,754 DEBUG [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T08:54:48,754 DEBUG [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T08:54:48,757 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(2659): reportForDuty to master=469387a2cdb6,33061,1732438487075 with port=34649, startcode=1732438487742 2024-11-24T08:54:48,757 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer(2659): reportForDuty to master=469387a2cdb6,33061,1732438487075 with port=32991, startcode=1732438487882 2024-11-24T08:54:48,759 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 469387a2cdb6,33061,1732438487075 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T08:54:48,766 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/469387a2cdb6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:54:48,767 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/469387a2cdb6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:54:48,767 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/469387a2cdb6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:54:48,767 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/469387a2cdb6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:54:48,767 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/469387a2cdb6:0, corePoolSize=10, maxPoolSize=10 2024-11-24T08:54:48,768 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,768 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/469387a2cdb6:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:54:48,768 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,768 DEBUG [RS:1;469387a2cdb6:32991 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T08:54:48,768 DEBUG [RS:0;469387a2cdb6:34649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T08:54:48,772 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:54:48,772 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T08:54:48,773 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732438518772 2024-11-24T08:54:48,775 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T08:54:48,776 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T08:54:48,778 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:48,778 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T08:54:48,780 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T08:54:48,780 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T08:54:48,781 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T08:54:48,781 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T08:54:48,782 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,785 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T08:54:48,787 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T08:54:48,787 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T08:54:48,789 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T08:54:48,790 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T08:54:48,791 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.large.0-1732438488791,5,FailOnTimeoutGroup] 2024-11-24T08:54:48,792 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.small.0-1732438488791,5,FailOnTimeoutGroup] 2024-11-24T08:54:48,792 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,792 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T08:54:48,793 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,793 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,808 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T08:54:48,809 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed 2024-11-24T08:54:48,820 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:48,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T08:54:48,831 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T08:54:48,832 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:48,832 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:48,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T08:54:48,842 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T08:54:48,843 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:48,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:48,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T08:54:48,846 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33125, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T08:54:48,846 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55579, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T08:54:48,850 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T08:54:48,851 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:48,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:48,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T08:54:48,854 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33061 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 469387a2cdb6,34649,1732438487742 2024-11-24T08:54:48,856 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33061 {}] master.ServerManager(517): Registering regionserver=469387a2cdb6,34649,1732438487742 2024-11-24T08:54:48,856 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T08:54:48,857 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:48,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:48,857 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T08:54:48,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740 2024-11-24T08:54:48,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740 2024-11-24T08:54:48,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T08:54:48,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T08:54:48,860 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T08:54:48,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T08:54:48,866 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33061 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 469387a2cdb6,32991,1732438487882 2024-11-24T08:54:48,866 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33061 {}] master.ServerManager(517): Registering regionserver=469387a2cdb6,32991,1732438487882 2024-11-24T08:54:48,869 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:54:48,869 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62767830, jitterRate=-0.06468644738197327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T08:54:48,871 DEBUG [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed 2024-11-24T08:54:48,871 DEBUG [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T08:54:48,871 DEBUG [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T08:54:48,874 DEBUG [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed 2024-11-24T08:54:48,874 DEBUG [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T08:54:48,874 DEBUG [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T08:54:48,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:54:48,875 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732438488820Initializing all the Stores at 1732438488821 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438488821Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438488826 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438488826Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438488826Cleaning up temporary data from old regions at 1732438488860 (+34 ms)Region opened successfully at 1732438488875 (+15 ms) 2024-11-24T08:54:48,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T08:54:48,876 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T08:54:48,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T08:54:48,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T08:54:48,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T08:54:48,878 DEBUG [RS:0;469387a2cdb6:34649 {}] zookeeper.ZKUtil(111): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/469387a2cdb6,34649,1732438487742 2024-11-24T08:54:48,878 WARN [RS:0;469387a2cdb6:34649 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:54:48,878 INFO [RS:0;469387a2cdb6:34649 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:54:48,879 DEBUG [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,34649,1732438487742 2024-11-24T08:54:48,879 DEBUG [RS:1;469387a2cdb6:32991 {}] zookeeper.ZKUtil(111): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/469387a2cdb6,32991,1732438487882 2024-11-24T08:54:48,879 WARN [RS:1;469387a2cdb6:32991 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:54:48,879 INFO [RS:1;469387a2cdb6:32991 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:54:48,880 DEBUG [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,32991,1732438487882 2024-11-24T08:54:48,880 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [469387a2cdb6,32991,1732438487882] 2024-11-24T08:54:48,880 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [469387a2cdb6,34649,1732438487742] 2024-11-24T08:54:48,881 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T08:54:48,881 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732438488876Disabling compacts and flushes for region at 1732438488876Disabling writes for close at 1732438488876Writing region close event to WAL at 1732438488880 (+4 ms)Closed at 1732438488881 (+1 ms) 2024-11-24T08:54:48,884 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:54:48,884 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T08:54:48,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T08:54:48,901 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T08:54:48,904 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T08:54:48,912 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T08:54:48,913 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T08:54:48,930 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T08:54:48,930 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T08:54:48,935 INFO [RS:0;469387a2cdb6:34649 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T08:54:48,936 INFO [RS:1;469387a2cdb6:32991 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T08:54:48,936 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,936 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,937 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T08:54:48,937 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T08:54:48,942 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T08:54:48,942 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T08:54:48,944 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,944 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,945 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,945 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,945 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,945 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,945 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,945 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,945 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,945 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,946 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,946 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,946 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/469387a2cdb6:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:54:48,946 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/469387a2cdb6:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:54:48,946 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,946 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,946 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,946 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,946 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,946 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,946 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,946 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,946 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,947 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,947 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/469387a2cdb6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:54:48,947 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,947 DEBUG [RS:1;469387a2cdb6:32991 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/469387a2cdb6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:54:48,947 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:48,947 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/469387a2cdb6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:54:48,947 DEBUG [RS:0;469387a2cdb6:34649 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/469387a2cdb6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:54:48,951 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,951 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,951 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,951 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,951 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,951 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,952 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,952 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,952 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,952 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,952 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,32991,1732438487882-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:54:48,952 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,34649,1732438487742-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:54:48,971 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T08:54:48,971 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T08:54:48,973 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,32991,1732438487882-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,973 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,34649,1732438487742-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,974 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,974 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,974 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.Replication(171): 469387a2cdb6,32991,1732438487882 started 2024-11-24T08:54:48,974 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.Replication(171): 469387a2cdb6,34649,1732438487742 started 2024-11-24T08:54:48,992 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,992 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:48,993 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer(1482): Serving as 469387a2cdb6,32991,1732438487882, RpcServer on 469387a2cdb6/172.17.0.2:32991, sessionid=0x10070f0cb510002 2024-11-24T08:54:48,993 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(1482): Serving as 469387a2cdb6,34649,1732438487742, RpcServer on 469387a2cdb6/172.17.0.2:34649, sessionid=0x10070f0cb510001 2024-11-24T08:54:48,994 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T08:54:48,994 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T08:54:48,994 DEBUG [RS:1;469387a2cdb6:32991 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 469387a2cdb6,32991,1732438487882 2024-11-24T08:54:48,994 DEBUG [RS:0;469387a2cdb6:34649 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 469387a2cdb6,34649,1732438487742 2024-11-24T08:54:48,994 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '469387a2cdb6,32991,1732438487882' 2024-11-24T08:54:48,994 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '469387a2cdb6,34649,1732438487742' 2024-11-24T08:54:48,994 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T08:54:48,994 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T08:54:48,995 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T08:54:48,995 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T08:54:48,996 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T08:54:48,996 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T08:54:48,996 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T08:54:48,996 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T08:54:48,996 DEBUG [RS:1;469387a2cdb6:32991 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 469387a2cdb6,32991,1732438487882 2024-11-24T08:54:48,996 DEBUG [RS:0;469387a2cdb6:34649 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 469387a2cdb6,34649,1732438487742 2024-11-24T08:54:48,996 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '469387a2cdb6,32991,1732438487882' 2024-11-24T08:54:48,996 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '469387a2cdb6,34649,1732438487742' 2024-11-24T08:54:48,996 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T08:54:48,996 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T08:54:48,997 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T08:54:48,997 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T08:54:48,997 DEBUG [RS:1;469387a2cdb6:32991 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T08:54:48,997 INFO [RS:1;469387a2cdb6:32991 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T08:54:48,997 DEBUG [RS:0;469387a2cdb6:34649 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T08:54:48,998 INFO [RS:1;469387a2cdb6:32991 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T08:54:48,998 INFO [RS:0;469387a2cdb6:34649 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T08:54:48,998 INFO [RS:0;469387a2cdb6:34649 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T08:54:49,055 WARN [469387a2cdb6:33061 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T08:54:49,102 INFO [RS:1;469387a2cdb6:32991 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:54:49,102 INFO [RS:0;469387a2cdb6:34649 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:54:49,103 INFO [RS:1;469387a2cdb6:32991 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=469387a2cdb6%2C32991%2C1732438487882, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,32991,1732438487882, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs, maxLogs=55 2024-11-24T08:54:49,103 INFO [RS:0;469387a2cdb6:34649 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=469387a2cdb6%2C34649%2C1732438487742, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,34649,1732438487742, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs, maxLogs=55 2024-11-24T08:54:49,122 INFO [RS:1;469387a2cdb6:32991 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,32991,1732438487882/469387a2cdb6%2C32991%2C1732438487882.1732438489106 2024-11-24T08:54:49,123 DEBUG [RS:1;469387a2cdb6:32991 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T08:54:49,124 INFO [RS:0;469387a2cdb6:34649 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,34649,1732438487742/469387a2cdb6%2C34649%2C1732438487742.1732438489106 2024-11-24T08:54:49,124 DEBUG [RS:0;469387a2cdb6:34649 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T08:54:49,310 DEBUG [469387a2cdb6:33061 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T08:54:49,323 DEBUG [469387a2cdb6:33061 {}] balancer.BalancerClusterState(204): Hosts are {469387a2cdb6=0} racks are {/default-rack=0} 2024-11-24T08:54:49,326 DEBUG [469387a2cdb6:33061 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T08:54:49,326 DEBUG [469387a2cdb6:33061 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T08:54:49,326 DEBUG [469387a2cdb6:33061 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T08:54:49,327 DEBUG [469387a2cdb6:33061 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T08:54:49,327 INFO [469387a2cdb6:33061 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T08:54:49,327 INFO [469387a2cdb6:33061 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T08:54:49,327 DEBUG [469387a2cdb6:33061 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T08:54:49,333 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=469387a2cdb6,34649,1732438487742 2024-11-24T08:54:49,337 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 469387a2cdb6,34649,1732438487742, state=OPENING 2024-11-24T08:54:49,341 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T08:54:49,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:49,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:49,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:49,344 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:49,344 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:49,344 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:49,345 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T08:54:49,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=469387a2cdb6,34649,1732438487742}] 2024-11-24T08:54:49,525 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T08:54:49,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56787, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T08:54:49,536 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T08:54:49,536 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:54:49,537 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T08:54:49,538 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=469387a2cdb6%2C34649%2C1732438487742.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,34649,1732438487742, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs, maxLogs=55 2024-11-24T08:54:49,557 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,34649,1732438487742/469387a2cdb6%2C34649%2C1732438487742.meta.1732438489540.meta 2024-11-24T08:54:49,558 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T08:54:49,558 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:54:49,560 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T08:54:49,564 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T08:54:49,569 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T08:54:49,572 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T08:54:49,573 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:49,573 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T08:54:49,573 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T08:54:49,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T08:54:49,575 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T08:54:49,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:49,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:49,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T08:54:49,577 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T08:54:49,577 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:49,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:49,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T08:54:49,578 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T08:54:49,578 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:49,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:49,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T08:54:49,579 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T08:54:49,579 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:49,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:49,579 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T08:54:49,580 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740 2024-11-24T08:54:49,581 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740 2024-11-24T08:54:49,581 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T08:54:49,581 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T08:54:49,582 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T08:54:49,583 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T08:54:49,583 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72772559, jitterRate=0.084395632147789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T08:54:49,583 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T08:54:49,586 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732438489574Writing region info on filesystem at 1732438489574Initializing all the Stores at 1732438489574Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438489574Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438489574Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438489575 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438489575Cleaning up temporary data from old regions at 1732438489581 (+6 ms)Running coprocessor post-open hooks at 1732438489583 (+2 ms)Region opened successfully at 1732438489585 (+2 ms) 2024-11-24T08:54:49,591 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732438489515 2024-11-24T08:54:49,600 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T08:54:49,601 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T08:54:49,602 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=469387a2cdb6,34649,1732438487742 2024-11-24T08:54:49,603 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 469387a2cdb6,34649,1732438487742, state=OPEN 2024-11-24T08:54:49,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:54:49,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:54:49,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:54:49,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:49,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:49,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:49,606 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=469387a2cdb6,34649,1732438487742 2024-11-24T08:54:49,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T08:54:49,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=469387a2cdb6,34649,1732438487742 in 259 msec 2024-11-24T08:54:49,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T08:54:49,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 720 msec 2024-11-24T08:54:49,616 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:54:49,616 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T08:54:49,633 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:54:49,635 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=469387a2cdb6,34649,1732438487742, seqNum=-1] 2024-11-24T08:54:49,652 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:54:49,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55833, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:54:49,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 990 msec 2024-11-24T08:54:49,676 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732438489676, completionTime=-1 2024-11-24T08:54:49,678 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T08:54:49,678 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T08:54:49,699 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T08:54:49,699 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732438549699 2024-11-24T08:54:49,699 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732438609699 2024-11-24T08:54:49,699 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 20 msec 2024-11-24T08:54:49,701 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T08:54:49,706 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,33061,1732438487075-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:49,707 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,33061,1732438487075-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:49,707 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,33061,1732438487075-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:49,708 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-469387a2cdb6:33061, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:49,708 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:49,709 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:49,714 DEBUG [master/469387a2cdb6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T08:54:49,732 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.749sec 2024-11-24T08:54:49,734 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T08:54:49,735 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T08:54:49,736 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T08:54:49,736 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T08:54:49,736 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T08:54:49,737 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,33061,1732438487075-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:54:49,737 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,33061,1732438487075-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T08:54:49,738 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T08:54:49,739 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T08:54:49,739 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,33061,1732438487075-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:49,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109a9e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:54:49,832 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T08:54:49,832 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T08:54:49,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 469387a2cdb6,33061,-1 for getting cluster id 2024-11-24T08:54:49,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:54:49,846 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13f39dd5-6acd-4f1f-9c5d-94e01df1863e' 2024-11-24T08:54:49,849 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:54:49,849 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13f39dd5-6acd-4f1f-9c5d-94e01df1863e" 2024-11-24T08:54:49,850 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@365ab761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:54:49,850 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [469387a2cdb6,33061,-1] 2024-11-24T08:54:49,852 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:54:49,854 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:49,855 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:54:49,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55969d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:54:49,859 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:54:49,866 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=469387a2cdb6,34649,1732438487742, seqNum=-1] 2024-11-24T08:54:49,866 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:54:49,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:54:49,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=469387a2cdb6,33061,1732438487075 2024-11-24T08:54:49,909 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T08:54:49,913 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 469387a2cdb6,33061,1732438487075 2024-11-24T08:54:49,916 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2a9c1634 2024-11-24T08:54:49,917 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T08:54:49,920 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T08:54:49,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33061 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732438489904', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:54:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33061 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732438489904 2024-11-24T08:54:49,932 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732438489904 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T08:54:49,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33061 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732438489904" procId is: 4 2024-11-24T08:54:49,935 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:49,937 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732438489904 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T08:54:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33061 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:54:49,949 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732438489904-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7ab09e538cf376c1be678ac60d09cb61, NAME => 'testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732438489904', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed 2024-11-24T08:54:49,955 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732438489904-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:49,955 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732438489904-pool-0 {}] regionserver.HRegion(1722): Closing 7ab09e538cf376c1be678ac60d09cb61, disabling compactions & flushes 2024-11-24T08:54:49,955 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732438489904-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:49,955 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732438489904-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:49,955 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732438489904-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. after waiting 0 ms 2024-11-24T08:54:49,956 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732438489904-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:49,956 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732438489904-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:49,956 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732438489904-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7ab09e538cf376c1be678ac60d09cb61: Waiting for close lock at 1732438489955Disabling compacts and flushes for region at 1732438489955Disabling writes for close at 1732438489956 (+1 ms)Writing region close event to WAL at 1732438489956Closed at 1732438489956 2024-11-24T08:54:49,957 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732438489904 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T08:54:49,962 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732438489957"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732438489957"}]},"ts":"1732438489957"} 2024-11-24T08:54:49,967 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T08:54:49,969 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732438489904 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T08:54:49,971 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732438489904","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732438489969"}]},"ts":"1732438489969"} 2024-11-24T08:54:49,975 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732438489904, state=ENABLING in hbase:meta 2024-11-24T08:54:49,976 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {469387a2cdb6=0} racks are {/default-rack=0} 2024-11-24T08:54:49,977 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T08:54:49,977 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T08:54:49,977 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T08:54:49,977 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T08:54:49,977 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T08:54:49,977 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T08:54:49,977 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T08:54:49,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732438489904, region=7ab09e538cf376c1be678ac60d09cb61, ASSIGN}] 2024-11-24T08:54:49,980 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732438489904, region=7ab09e538cf376c1be678ac60d09cb61, ASSIGN 2024-11-24T08:54:49,981 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732438489904, region=7ab09e538cf376c1be678ac60d09cb61, ASSIGN; state=OFFLINE, location=469387a2cdb6,32991,1732438487882; forceNewPlan=false, retain=false 2024-11-24T08:54:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33061 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:54:50,135 INFO [469387a2cdb6:33061 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T08:54:50,136 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7ab09e538cf376c1be678ac60d09cb61, regionState=OPENING, regionLocation=469387a2cdb6,32991,1732438487882 2024-11-24T08:54:50,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732438489904, region=7ab09e538cf376c1be678ac60d09cb61, ASSIGN because future has completed 2024-11-24T08:54:50,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ab09e538cf376c1be678ac60d09cb61, server=469387a2cdb6,32991,1732438487882}] 2024-11-24T08:54:50,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33061 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:54:50,297 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T08:54:50,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48231, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T08:54:50,308 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:50,309 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ab09e538cf376c1be678ac60d09cb61, NAME => 'testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:54:50,310 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732438489904 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:50,310 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:50,310 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:50,310 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:50,311 INFO [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:50,315 INFO [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ab09e538cf376c1be678ac60d09cb61 columnFamilyName family 2024-11-24T08:54:50,315 DEBUG [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:50,316 INFO [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] regionserver.HStore(327): Store=7ab09e538cf376c1be678ac60d09cb61/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:54:50,316 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:50,317 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:50,317 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:50,317 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:50,317 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:50,318 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:50,324 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:54:50,324 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7ab09e538cf376c1be678ac60d09cb61; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64479217, jitterRate=-0.03918479382991791}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:54:50,325 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:50,325 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7ab09e538cf376c1be678ac60d09cb61: Running coprocessor pre-open hook at 1732438490310Writing region info on filesystem at 1732438490310Initializing all the Stores at 1732438490311 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438490311Cleaning up temporary data from old regions at 1732438490317 (+6 ms)Running coprocessor post-open hooks at 1732438490325 (+8 ms)Region opened successfully at 1732438490325 2024-11-24T08:54:50,326 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61., pid=6, masterSystemTime=1732438490297 2024-11-24T08:54:50,330 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:50,330 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:50,330 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7ab09e538cf376c1be678ac60d09cb61, regionState=OPEN, openSeqNum=2, regionLocation=469387a2cdb6,32991,1732438487882 2024-11-24T08:54:50,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ab09e538cf376c1be678ac60d09cb61, server=469387a2cdb6,32991,1732438487882 because future has completed 2024-11-24T08:54:50,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T08:54:50,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ab09e538cf376c1be678ac60d09cb61, server=469387a2cdb6,32991,1732438487882 in 192 msec 2024-11-24T08:54:50,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T08:54:50,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732438489904, region=7ab09e538cf376c1be678ac60d09cb61, ASSIGN in 359 msec 2024-11-24T08:54:50,340 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732438489904 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T08:54:50,340 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732438489904","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732438490340"}]},"ts":"1732438490340"} 2024-11-24T08:54:50,343 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732438489904, state=ENABLED in hbase:meta 2024-11-24T08:54:50,344 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732438489904 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T08:54:50,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732438489904 in 418 msec 2024-11-24T08:54:50,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33061 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:54:50,576 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732438489904 completed 2024-11-24T08:54:50,576 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732438489904 get assigned. Timeout = 60000ms 2024-11-24T08:54:50,580 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T08:54:50,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732438489904 assigned to meta. Checking AM states. 2024-11-24T08:54:50,588 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T08:54:50,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732438489904 assigned. 2024-11-24T08:54:50,597 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732438489904', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61., hostname=469387a2cdb6,32991,1732438487882, seqNum=2] 2024-11-24T08:54:50,598 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:54:50,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:54:50,610 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7ab09e538cf376c1be678ac60d09cb61 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T08:54:50,633 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c012eea312aa42d7985bf4c368b4e0d1 2024-11-24T08:54:50,668 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/.tmp/family/c012eea312aa42d7985bf4c368b4e0d1 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T08:54:50,673 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/.tmp/family/c012eea312aa42d7985bf4c368b4e0d1 2024-11-24T08:54:50,704 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/.tmp/family/c012eea312aa42d7985bf4c368b4e0d1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1 2024-11-24T08:54:50,706 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1, delay=922 ms 2024-11-24T08:54:50,707 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T08:54:50,713 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 7ab09e538cf376c1be678ac60d09cb61 in 102ms, sequenceid=6, compaction requested=false 2024-11-24T08:54:50,714 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732438489904' 2024-11-24T08:54:50,716 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7ab09e538cf376c1be678ac60d09cb61: 2024-11-24T08:54:51,631 DEBUG [hfile-prefetch-1732438490706 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key c012eea312aa42d7985bf4c368b4e0d1_0. 2024-11-24T08:54:51,639 DEBUG [hfile-prefetch-1732438490706 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key c012eea312aa42d7985bf4c368b4e0d1_117. 2024-11-24T08:54:51,640 DEBUG [hfile-prefetch-1732438490706 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1, with total blocks 2, and data blocks 1 2024-11-24T08:54:51,640 DEBUG [hfile-prefetch-1732438490706 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T08:54:51,640 DEBUG [hfile-prefetch-1732438490706 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T08:54:51,641 DEBUG [hfile-prefetch-1732438490706 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1. Will sleep 100ms and try the verification again. 2024-11-24T08:54:51,702 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T08:54:51,702 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T08:54:51,703 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T08:54:51,703 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T08:54:51,726 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T08:54:51,726 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T08:54:51,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33061 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=7ab09e538cf376c1be678ac60d09cb61, source=469387a2cdb6,32991,1732438487882, destination=469387a2cdb6,34649,1732438487742, warming up region on 469387a2cdb6,34649,1732438487742 2024-11-24T08:54:51,727 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T08:54:51,727 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T08:54:51,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33061 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=7ab09e538cf376c1be678ac60d09cb61, source=469387a2cdb6,32991,1732438487882, destination=469387a2cdb6,34649,1732438487742, running balancer 2024-11-24T08:54:51,729 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-24T08:54:51,729 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732438491702 2024-11-24T08:54:51,729 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T08:54:51,730 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T08:54:51,730 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732438491726 2024-11-24T08:54:51,730 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T08:54:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33061 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732438489904, region=7ab09e538cf376c1be678ac60d09cb61, REOPEN/MOVE 2024-11-24T08:54:51,730 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732438489904, region=7ab09e538cf376c1be678ac60d09cb61, REOPEN/MOVE 2024-11-24T08:54:51,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34649 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34649 {}] regionserver.HRegion(7855): Warmup {ENCODED => 7ab09e538cf376c1be678ac60d09cb61, NAME => 'testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:54:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34649 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:51,732 INFO [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:51,732 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7ab09e538cf376c1be678ac60d09cb61, regionState=CLOSING, regionLocation=469387a2cdb6,32991,1732438487882 2024-11-24T08:54:51,733 INFO [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ab09e538cf376c1be678ac60d09cb61 columnFamilyName family 2024-11-24T08:54:51,733 DEBUG [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:51,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732438489904, region=7ab09e538cf376c1be678ac60d09cb61, REOPEN/MOVE because future has completed 2024-11-24T08:54:51,736 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-24T08:54:51,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7ab09e538cf376c1be678ac60d09cb61, server=469387a2cdb6,32991,1732438487882}] 2024-11-24T08:54:51,737 DEBUG [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1 2024-11-24T08:54:51,737 INFO [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] regionserver.HStore(327): Store=7ab09e538cf376c1be678ac60d09cb61/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:54:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34649 {}] regionserver.HRegion(1722): Closing 7ab09e538cf376c1be678ac60d09cb61, disabling compactions & flushes 2024-11-24T08:54:51,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34649 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34649 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34649 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. after waiting 0 ms 2024-11-24T08:54:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34649 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:51,739 DEBUG [StoreFileCloser-7ab09e538cf376c1be678ac60d09cb61-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1 2024-11-24T08:54:51,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34649 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34649 {}] regionserver.HRegion(1676): Region close journal for 7ab09e538cf376c1be678ac60d09cb61: Waiting for close lock at 1732438491737Disabling compacts and flushes for region at 1732438491737Disabling writes for close at 1732438491738 (+1 ms)Writing region close event to WAL at 1732438491739 (+1 ms)Closed at 1732438491739 2024-11-24T08:54:51,741 WARN [hfile-prefetch-1732438490706 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-24T08:54:51,742 DEBUG [hfile-prefetch-1732438490706 {}] hfile.PrefetchExecutor(121): Prefetch completed for c012eea312aa42d7985bf4c368b4e0d1 2024-11-24T08:54:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33061 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T08:54:51,896 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:51,896 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-24T08:54:51,897 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 7ab09e538cf376c1be678ac60d09cb61, disabling compactions & flushes 2024-11-24T08:54:51,897 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:51,897 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:51,897 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. after waiting 0 ms 2024-11-24T08:54:51,897 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:51,899 DEBUG [StoreFileCloser-7ab09e538cf376c1be678ac60d09cb61-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file c012eea312aa42d7985bf4c368b4e0d1, starting offset: 0, end offset: 9223372036854775807 2024-11-24T08:54:51,901 DEBUG [StoreFileCloser-7ab09e538cf376c1be678ac60d09cb61-family-1 {}] bucket.BucketCache(856): removed key c012eea312aa42d7985bf4c368b4e0d1_0 from back map with offset lock 2094080 in the evict process 2024-11-24T08:54:51,907 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T08:54:51,908 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:51,909 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 7ab09e538cf376c1be678ac60d09cb61: Waiting for close lock at 1732438491897Running coprocessor pre-close hooks at 1732438491897Disabling compacts and flushes for region at 1732438491897Disabling writes for close at 1732438491897Writing region close event to WAL at 1732438491902 (+5 ms)Running coprocessor post-close hooks at 1732438491907 (+5 ms)Closed at 1732438491908 (+1 ms) 2024-11-24T08:54:51,909 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 7ab09e538cf376c1be678ac60d09cb61 move to 469387a2cdb6,34649,1732438487742 record at close sequenceid=6 2024-11-24T08:54:51,913 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:51,913 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7ab09e538cf376c1be678ac60d09cb61, regionState=CLOSED 2024-11-24T08:54:51,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7ab09e538cf376c1be678ac60d09cb61, server=469387a2cdb6,32991,1732438487882 because future has completed 2024-11-24T08:54:51,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T08:54:51,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 7ab09e538cf376c1be678ac60d09cb61, server=469387a2cdb6,32991,1732438487882 in 181 msec 2024-11-24T08:54:51,922 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732438489904, region=7ab09e538cf376c1be678ac60d09cb61, REOPEN/MOVE; state=CLOSED, location=469387a2cdb6,34649,1732438487742; forceNewPlan=false, retain=false 2024-11-24T08:54:51,929 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T08:54:51,929 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T08:54:51,930 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T08:54:51,930 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T08:54:51,930 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T08:54:51,931 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T08:54:51,931 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T08:54:51,931 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T08:54:51,932 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T08:54:51,932 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732438491929 2024-11-24T08:54:51,932 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T08:54:51,933 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T08:54:51,934 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732438491931 2024-11-24T08:54:51,934 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T08:54:52,073 INFO [469387a2cdb6:33061 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T08:54:52,074 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7ab09e538cf376c1be678ac60d09cb61, regionState=OPENING, regionLocation=469387a2cdb6,34649,1732438487742 2024-11-24T08:54:52,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732438489904, region=7ab09e538cf376c1be678ac60d09cb61, REOPEN/MOVE because future has completed 2024-11-24T08:54:52,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ab09e538cf376c1be678ac60d09cb61, server=469387a2cdb6,34649,1732438487742}] 2024-11-24T08:54:52,239 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:52,240 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ab09e538cf376c1be678ac60d09cb61, NAME => 'testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:54:52,241 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732438489904 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,241 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:52,242 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,242 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,244 INFO [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,245 INFO [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ab09e538cf376c1be678ac60d09cb61 columnFamilyName family 2024-11-24T08:54:52,245 DEBUG [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:52,247 DEBUG [StoreFileOpener-7ab09e538cf376c1be678ac60d09cb61-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1, delay=959 ms 2024-11-24T08:54:52,248 DEBUG [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1 2024-11-24T08:54:52,248 INFO [StoreOpener-7ab09e538cf376c1be678ac60d09cb61-1 {}] regionserver.HStore(327): Store=7ab09e538cf376c1be678ac60d09cb61/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:54:52,248 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,250 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,251 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,251 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,251 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,256 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,256 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 7ab09e538cf376c1be678ac60d09cb61; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68595597, jitterRate=0.022154048085212708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:54:52,257 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,258 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 7ab09e538cf376c1be678ac60d09cb61: Running coprocessor pre-open hook at 1732438492242Writing region info on filesystem at 1732438492242Initializing all the Stores at 1732438492243 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438492243Cleaning up temporary data from old regions at 1732438492251 (+8 ms)Running coprocessor post-open hooks at 1732438492257 (+6 ms)Region opened successfully at 1732438492258 (+1 ms) 2024-11-24T08:54:52,259 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61., pid=9, masterSystemTime=1732438492233 2024-11-24T08:54:52,261 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:52,261 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:52,262 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7ab09e538cf376c1be678ac60d09cb61, regionState=OPEN, openSeqNum=10, regionLocation=469387a2cdb6,34649,1732438487742 2024-11-24T08:54:52,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ab09e538cf376c1be678ac60d09cb61, server=469387a2cdb6,34649,1732438487742 because future has completed 2024-11-24T08:54:52,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T08:54:52,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ab09e538cf376c1be678ac60d09cb61, server=469387a2cdb6,34649,1732438487742 in 186 msec 2024-11-24T08:54:52,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732438489904, region=7ab09e538cf376c1be678ac60d09cb61, REOPEN/MOVE in 539 msec 2024-11-24T08:54:52,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T08:54:52,284 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:54:52,284 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:54:52,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:52,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:52,289 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:54:52,289 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T08:54:52,289 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-11-24T08:54:52,290 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=469387a2cdb6,33061,1732438487075 2024-11-24T08:54:52,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:52,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:52,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:52,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:52,292 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T08:54:52,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:52,292 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:54:52,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:52,293 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:54:52,293 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:52,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:52,293 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:52,293 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:52,294 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '469387a2cdb6,34649,1732438487742' ***** 2024-11-24T08:54:52,294 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T08:54:52,294 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '469387a2cdb6,32991,1732438487882' ***** 2024-11-24T08:54:52,294 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T08:54:52,294 INFO [RS:0;469387a2cdb6:34649 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T08:54:52,294 INFO [RS:1;469387a2cdb6:32991 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T08:54:52,295 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T08:54:52,295 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T08:54:52,296 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T08:54:52,296 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T08:54:52,296 INFO [RS:1;469387a2cdb6:32991 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/bucket.persistence 2024-11-24T08:54:52,296 INFO [RS:0;469387a2cdb6:34649 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/bucket.persistence 2024-11-24T08:54:52,296 INFO [RS:0;469387a2cdb6:34649 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T08:54:52,296 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T08:54:52,297 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T08:54:52,297 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T08:54:52,297 INFO [RS:1;469387a2cdb6:32991 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T08:54:52,297 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T08:54:52,307 DEBUG [RS:0;469387a2cdb6:34649 {}] bucket.BucketCache(1487): Thread RS:0;469387a2cdb6:34649 started persisting bucket cache to file 2024-11-24T08:54:52,307 DEBUG [RS:0;469387a2cdb6:34649 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T08:54:52,307 DEBUG [RS:0;469387a2cdb6:34649 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T08:54:52,307 DEBUG [RS:1;469387a2cdb6:32991 {}] bucket.BucketCache(1487): Thread RS:1;469387a2cdb6:32991 started persisting bucket cache to file 2024-11-24T08:54:52,307 DEBUG [RS:1;469387a2cdb6:32991 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T08:54:52,307 DEBUG [RS:1;469387a2cdb6:32991 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T08:54:52,310 DEBUG [RS:0;469387a2cdb6:34649 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T08:54:52,310 DEBUG [RS:1;469387a2cdb6:32991 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T08:54:52,310 DEBUG [RS:1;469387a2cdb6:32991 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732438492307 2024-11-24T08:54:52,310 DEBUG [RS:0;469387a2cdb6:34649 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732438492307 2024-11-24T08:54:52,310 DEBUG [RS:1;469387a2cdb6:32991 {}] bucket.BucketCache(1509): Thread RS:1;469387a2cdb6:32991 finished persisting bucket cache to file, renaming 2024-11-24T08:54:52,310 DEBUG [RS:0;469387a2cdb6:34649 {}] bucket.BucketCache(1509): Thread RS:0;469387a2cdb6:34649 finished persisting bucket cache to file, renaming 2024-11-24T08:54:52,310 WARN [RS:0;469387a2cdb6:34649 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-24T08:54:52,310 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T08:54:52,310 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T08:54:52,311 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T08:54:52,311 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T08:54:52,311 INFO [RS:1;469387a2cdb6:32991 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T08:54:52,311 INFO [RS:0;469387a2cdb6:34649 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T08:54:52,311 INFO [RS:0;469387a2cdb6:34649 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T08:54:52,311 INFO [RS:1;469387a2cdb6:32991 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T08:54:52,311 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer(959): stopping server 469387a2cdb6,32991,1732438487882 2024-11-24T08:54:52,311 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(3091): Received CLOSE for 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,311 INFO [RS:1;469387a2cdb6:32991 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:54:52,311 INFO [RS:1;469387a2cdb6:32991 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;469387a2cdb6:32991. 2024-11-24T08:54:52,311 DEBUG [RS:1;469387a2cdb6:32991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:54:52,312 DEBUG [RS:1;469387a2cdb6:32991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:52,312 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(959): stopping server 469387a2cdb6,34649,1732438487742 2024-11-24T08:54:52,312 INFO [RS:0;469387a2cdb6:34649 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:54:52,312 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer(976): stopping server 469387a2cdb6,32991,1732438487882; all regions closed. 2024-11-24T08:54:52,312 INFO [RS:0;469387a2cdb6:34649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;469387a2cdb6:34649. 2024-11-24T08:54:52,312 DEBUG [RS:0;469387a2cdb6:34649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:54:52,312 DEBUG [RS:0;469387a2cdb6:34649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:52,312 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T08:54:52,312 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T08:54:52,312 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T08:54:52,312 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T08:54:52,312 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7ab09e538cf376c1be678ac60d09cb61, disabling compactions & flushes 2024-11-24T08:54:52,313 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T08:54:52,313 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:52,313 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:52,313 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. after waiting 0 ms 2024-11-24T08:54:52,313 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:52,313 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T08:54:52,313 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T08:54:52,313 DEBUG [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 7ab09e538cf376c1be678ac60d09cb61=testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61.} 2024-11-24T08:54:52,313 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T08:54:52,313 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T08:54:52,313 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T08:54:52,313 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T08:54:52,314 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T08:54:52,314 DEBUG [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7ab09e538cf376c1be678ac60d09cb61 2024-11-24T08:54:52,314 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-24T08:54:52,316 DEBUG [StoreFileCloser-7ab09e538cf376c1be678ac60d09cb61-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/family/c012eea312aa42d7985bf4c368b4e0d1 2024-11-24T08:54:52,320 DEBUG [RS:1;469387a2cdb6:32991 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs 2024-11-24T08:54:52,320 INFO [RS:1;469387a2cdb6:32991 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 469387a2cdb6%2C32991%2C1732438487882:(num 1732438489106) 2024-11-24T08:54:52,320 DEBUG [RS:1;469387a2cdb6:32991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:52,320 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:54:52,320 INFO [RS:1;469387a2cdb6:32991 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:54:52,321 INFO [RS:1;469387a2cdb6:32991 {}] hbase.ChoreService(370): Chore service for: regionserver/469387a2cdb6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T08:54:52,321 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T08:54:52,321 INFO [regionserver/469387a2cdb6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:54:52,321 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T08:54:52,321 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T08:54:52,321 INFO [RS:1;469387a2cdb6:32991 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:54:52,322 INFO [RS:1;469387a2cdb6:32991 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32991 2024-11-24T08:54:52,323 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnRegionMove1732438489904/7ab09e538cf376c1be678ac60d09cb61/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T08:54:52,324 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:52,324 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7ab09e538cf376c1be678ac60d09cb61: Waiting for close lock at 1732438492312Running coprocessor pre-close hooks at 1732438492312Disabling compacts and flushes for region at 1732438492312Disabling writes for close at 1732438492313 (+1 ms)Writing region close event to WAL at 1732438492317 (+4 ms)Running coprocessor post-close hooks at 1732438492324 (+7 ms)Closed at 1732438492324 2024-11-24T08:54:52,325 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61. 2024-11-24T08:54:52,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/469387a2cdb6,32991,1732438487882 2024-11-24T08:54:52,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:54:52,325 INFO [RS:1;469387a2cdb6:32991 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:54:52,327 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 944e26d7ab4e47b9ab8a1bf58614c94e 2024-11-24T08:54:52,327 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [469387a2cdb6,32991,1732438487882] 2024-11-24T08:54:52,328 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/469387a2cdb6,32991,1732438487882 already deleted, retry=false 2024-11-24T08:54:52,328 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 469387a2cdb6,32991,1732438487882 expired; onlineServers=1 2024-11-24T08:54:52,344 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/info/944e26d7ab4e47b9ab8a1bf58614c94e is 205, key is testBlockEvictionOnRegionMove1732438489904,,1732438489921.7ab09e538cf376c1be678ac60d09cb61./info:regioninfo/1732438492261/Put/seqid=0 2024-11-24T08:54:52,345 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/info/944e26d7ab4e47b9ab8a1bf58614c94e 2024-11-24T08:54:52,354 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f17c1f55359b4aef9019ecbe81ec6099 2024-11-24T08:54:52,355 INFO [regionserver/469387a2cdb6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:54:52,358 INFO [regionserver/469387a2cdb6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:54:52,369 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/ns/f17c1f55359b4aef9019ecbe81ec6099 is 43, key is default/ns:d/1732438489659/Put/seqid=0 2024-11-24T08:54:52,370 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/ns/f17c1f55359b4aef9019ecbe81ec6099 2024-11-24T08:54:52,380 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cb4dcd881d544e20809fc7dd9b0bff1c 2024-11-24T08:54:52,393 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/table/cb4dcd881d544e20809fc7dd9b0bff1c is 78, key is testBlockEvictionOnRegionMove1732438489904/table:state/1732438490340/Put/seqid=0 2024-11-24T08:54:52,394 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/table/cb4dcd881d544e20809fc7dd9b0bff1c 2024-11-24T08:54:52,397 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/info/944e26d7ab4e47b9ab8a1bf58614c94e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/info/944e26d7ab4e47b9ab8a1bf58614c94e 2024-11-24T08:54:52,399 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/info/944e26d7ab4e47b9ab8a1bf58614c94e, delay=967 ms 2024-11-24T08:54:52,401 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/info/944e26d7ab4e47b9ab8a1bf58614c94e, entries=15, sequenceid=15, filesize=7.6 K 2024-11-24T08:54:52,402 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/ns/f17c1f55359b4aef9019ecbe81ec6099 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/ns/f17c1f55359b4aef9019ecbe81ec6099 2024-11-24T08:54:52,403 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/ns/f17c1f55359b4aef9019ecbe81ec6099, delay=950 ms 2024-11-24T08:54:52,405 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/ns/f17c1f55359b4aef9019ecbe81ec6099, entries=2, sequenceid=15, filesize=5.0 K 2024-11-24T08:54:52,405 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/table/cb4dcd881d544e20809fc7dd9b0bff1c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/table/cb4dcd881d544e20809fc7dd9b0bff1c 2024-11-24T08:54:52,407 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/table/cb4dcd881d544e20809fc7dd9b0bff1c, delay=967 ms 2024-11-24T08:54:52,408 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/table/cb4dcd881d544e20809fc7dd9b0bff1c, entries=2, sequenceid=15, filesize=5.3 K 2024-11-24T08:54:52,409 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=15, compaction requested=false 2024-11-24T08:54:52,410 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T08:54:52,411 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/info/944e26d7ab4e47b9ab8a1bf58614c94e 2024-11-24T08:54:52,412 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/ns/f17c1f55359b4aef9019ecbe81ec6099 2024-11-24T08:54:52,412 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/table/cb4dcd881d544e20809fc7dd9b0bff1c 2024-11-24T08:54:52,416 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-24T08:54:52,416 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T08:54:52,417 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T08:54:52,417 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732438492313Running coprocessor pre-close hooks at 1732438492313Disabling compacts and flushes for region at 1732438492313Disabling writes for close at 1732438492314 (+1 ms)Obtaining lock to block concurrent updates at 1732438492314Preparing flush snapshotting stores in 1588230740 at 1732438492314Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732438492315 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732438492315Flushing 1588230740/info: creating writer at 1732438492315Flushing 1588230740/info: appending metadata at 1732438492343 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732438492344 (+1 ms)Flushing 1588230740/ns: creating writer at 1732438492348 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732438492368 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732438492368Flushing 1588230740/table: creating writer at 1732438492374 (+6 ms)Flushing 1588230740/table: appending metadata at 1732438492392 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732438492392Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@111b9e8e: reopening flushed file at 1732438492397 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1822f92: reopening flushed file at 1732438492401 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dc9ed18: reopening flushed file at 1732438492405 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=15, compaction requested=false at 1732438492410 (+5 ms)Writing region close event to WAL at 1732438492412 (+2 ms)Running coprocessor post-close hooks at 1732438492416 (+4 ms)Closed at 1732438492417 (+1 ms) 2024-11-24T08:54:52,417 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T08:54:52,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:52,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32991-0x10070f0cb510002, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:52,427 INFO [RS:1;469387a2cdb6:32991 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:54:52,428 INFO [RS:1;469387a2cdb6:32991 {}] regionserver.HRegionServer(1031): Exiting; stopping=469387a2cdb6,32991,1732438487882; zookeeper connection closed. 2024-11-24T08:54:52,428 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3657ce9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3657ce9 2024-11-24T08:54:52,514 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(976): stopping server 469387a2cdb6,34649,1732438487742; all regions closed. 2024-11-24T08:54:52,521 DEBUG [RS:0;469387a2cdb6:34649 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs 2024-11-24T08:54:52,521 INFO [RS:0;469387a2cdb6:34649 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 469387a2cdb6%2C34649%2C1732438487742.meta:.meta(num 1732438489540) 2024-11-24T08:54:52,524 DEBUG [RS:0;469387a2cdb6:34649 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs 2024-11-24T08:54:52,524 INFO [RS:0;469387a2cdb6:34649 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 469387a2cdb6%2C34649%2C1732438487742:(num 1732438489106) 2024-11-24T08:54:52,525 DEBUG [RS:0;469387a2cdb6:34649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:52,525 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:54:52,525 INFO [RS:0;469387a2cdb6:34649 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:54:52,525 INFO [RS:0;469387a2cdb6:34649 {}] hbase.ChoreService(370): Chore service for: regionserver/469387a2cdb6:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T08:54:52,525 INFO [RS:0;469387a2cdb6:34649 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:54:52,525 INFO [regionserver/469387a2cdb6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:54:52,526 INFO [RS:0;469387a2cdb6:34649 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34649 2024-11-24T08:54:52,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:54:52,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/469387a2cdb6,34649,1732438487742 2024-11-24T08:54:52,528 INFO [RS:0;469387a2cdb6:34649 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:54:52,528 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [469387a2cdb6,34649,1732438487742] 2024-11-24T08:54:52,529 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/469387a2cdb6,34649,1732438487742 already deleted, retry=false 2024-11-24T08:54:52,530 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 469387a2cdb6,34649,1732438487742 expired; onlineServers=0 2024-11-24T08:54:52,530 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '469387a2cdb6,33061,1732438487075' ***** 2024-11-24T08:54:52,530 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T08:54:52,530 INFO [M:0;469387a2cdb6:33061 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:54:52,530 INFO [M:0;469387a2cdb6:33061 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:54:52,530 DEBUG [M:0;469387a2cdb6:33061 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T08:54:52,530 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T08:54:52,531 DEBUG [M:0;469387a2cdb6:33061 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T08:54:52,531 DEBUG [master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.small.0-1732438488791 {}] cleaner.HFileCleaner(306): Exit Thread[master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.small.0-1732438488791,5,FailOnTimeoutGroup] 2024-11-24T08:54:52,531 DEBUG [master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.large.0-1732438488791 {}] cleaner.HFileCleaner(306): Exit Thread[master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.large.0-1732438488791,5,FailOnTimeoutGroup] 2024-11-24T08:54:52,531 INFO [M:0;469387a2cdb6:33061 {}] hbase.ChoreService(370): Chore service for: master/469387a2cdb6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T08:54:52,531 INFO [M:0;469387a2cdb6:33061 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:54:52,531 DEBUG [M:0;469387a2cdb6:33061 {}] master.HMaster(1795): Stopping service threads 2024-11-24T08:54:52,531 INFO [M:0;469387a2cdb6:33061 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T08:54:52,532 INFO [M:0;469387a2cdb6:33061 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T08:54:52,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T08:54:52,532 INFO [M:0;469387a2cdb6:33061 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T08:54:52,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:52,532 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T08:54:52,533 DEBUG [M:0;469387a2cdb6:33061 {}] zookeeper.ZKUtil(347): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T08:54:52,533 WARN [M:0;469387a2cdb6:33061 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T08:54:52,533 INFO [M:0;469387a2cdb6:33061 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/.lastflushedseqids 2024-11-24T08:54:52,543 INFO [M:0;469387a2cdb6:33061 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T08:54:52,543 INFO [M:0;469387a2cdb6:33061 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T08:54:52,543 DEBUG [M:0;469387a2cdb6:33061 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T08:54:52,543 INFO [M:0;469387a2cdb6:33061 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:52,543 DEBUG [M:0;469387a2cdb6:33061 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:52,544 DEBUG [M:0;469387a2cdb6:33061 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T08:54:52,544 DEBUG [M:0;469387a2cdb6:33061 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:52,544 INFO [M:0;469387a2cdb6:33061 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-24T08:54:52,552 DEBUG [M:0;469387a2cdb6:33061 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2e9af41e863f4e819fab56652c8bbba7 2024-11-24T08:54:52,564 DEBUG [M:0;469387a2cdb6:33061 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2e9af41e863f4e819fab56652c8bbba7 is 82, key is hbase:meta,,1/info:regioninfo/1732438489601/Put/seqid=0 2024-11-24T08:54:52,565 INFO [M:0;469387a2cdb6:33061 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2e9af41e863f4e819fab56652c8bbba7 2024-11-24T08:54:52,572 DEBUG [M:0;469387a2cdb6:33061 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c9923ae9473645fa96803684adf90edc 2024-11-24T08:54:52,585 DEBUG [M:0;469387a2cdb6:33061 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c9923ae9473645fa96803684adf90edc is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732438490346/Put/seqid=0 2024-11-24T08:54:52,586 INFO [M:0;469387a2cdb6:33061 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c9923ae9473645fa96803684adf90edc 2024-11-24T08:54:52,593 DEBUG [M:0;469387a2cdb6:33061 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fd0bf8327d29474a8afdbb6d902d3131 2024-11-24T08:54:52,606 DEBUG [M:0;469387a2cdb6:33061 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd0bf8327d29474a8afdbb6d902d3131 is 69, key is 469387a2cdb6,32991,1732438487882/rs:state/1732438488867/Put/seqid=0 2024-11-24T08:54:52,606 INFO [M:0;469387a2cdb6:33061 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd0bf8327d29474a8afdbb6d902d3131 2024-11-24T08:54:52,608 DEBUG [M:0;469387a2cdb6:33061 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2e9af41e863f4e819fab56652c8bbba7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2e9af41e863f4e819fab56652c8bbba7 2024-11-24T08:54:52,610 INFO [M:0;469387a2cdb6:33061 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2e9af41e863f4e819fab56652c8bbba7, entries=8, sequenceid=84, filesize=5.5 K 2024-11-24T08:54:52,610 DEBUG [M:0;469387a2cdb6:33061 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c9923ae9473645fa96803684adf90edc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c9923ae9473645fa96803684adf90edc 2024-11-24T08:54:52,612 INFO [M:0;469387a2cdb6:33061 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c9923ae9473645fa96803684adf90edc, entries=9, sequenceid=84, filesize=6.5 K 2024-11-24T08:54:52,612 DEBUG [M:0;469387a2cdb6:33061 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd0bf8327d29474a8afdbb6d902d3131 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fd0bf8327d29474a8afdbb6d902d3131 2024-11-24T08:54:52,614 INFO [M:0;469387a2cdb6:33061 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fd0bf8327d29474a8afdbb6d902d3131, entries=2, sequenceid=84, filesize=5.1 K 2024-11-24T08:54:52,615 INFO [M:0;469387a2cdb6:33061 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=84, compaction requested=false 2024-11-24T08:54:52,617 INFO [M:0;469387a2cdb6:33061 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:52,617 DEBUG [M:0;469387a2cdb6:33061 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732438492543Disabling compacts and flushes for region at 1732438492543Disabling writes for close at 1732438492544 (+1 ms)Obtaining lock to block concurrent updates at 1732438492544Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732438492544Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732438492544Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732438492545 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732438492545Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732438492564 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732438492564Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732438492567 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732438492585 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732438492585Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732438492588 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732438492605 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732438492605Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d2c9de1: reopening flushed file at 1732438492608 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dc92a2e: reopening flushed file at 1732438492610 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8812337: reopening flushed file at 1732438492612 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=84, compaction requested=false at 1732438492615 (+3 ms)Writing region close event to WAL at 1732438492617 (+2 ms)Closed at 1732438492617 2024-11-24T08:54:52,619 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:54:52,619 INFO [M:0;469387a2cdb6:33061 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T08:54:52,619 INFO [M:0;469387a2cdb6:33061 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33061 2024-11-24T08:54:52,619 INFO [M:0;469387a2cdb6:33061 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:54:52,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:52,629 INFO [RS:0;469387a2cdb6:34649 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:54:52,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34649-0x10070f0cb510001, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:52,629 INFO [RS:0;469387a2cdb6:34649 {}] regionserver.HRegionServer(1031): Exiting; stopping=469387a2cdb6,34649,1732438487742; zookeeper connection closed. 2024-11-24T08:54:52,629 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@60d6ef7e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@60d6ef7e 2024-11-24T08:54:52,630 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-24T08:54:52,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:52,721 INFO [M:0;469387a2cdb6:33061 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:54:52,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33061-0x10070f0cb510000, quorum=127.0.0.1:64764, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:52,728 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T08:54:52,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T08:54:52,749 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:52,756 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/469387a2cdb6:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/469387a2cdb6:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/469387a2cdb6:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732438492399 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732438492247 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732438492403 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/469387a2cdb6:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1732438490706 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 111) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2222 (was 2569) 2024-11-24T08:54:52,761 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=2222 2024-11-24T08:54:52,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/hadoop-log-dir so I do NOT create it in target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588 2024-11-24T08:54:52,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/hadoop-tmp-dir so I do NOT create it in target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588 2024-11-24T08:54:52,762 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:52,764 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/cluster_a4adec14-55dc-2f49-b4d3-4a658f065cad, deleteOnExit=true 2024-11-24T08:54:52,767 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/cluster_a4adec14-55dc-2f49-b4d3-4a658f065cad/zookeeper_0, clientPort=62639, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/cluster_a4adec14-55dc-2f49-b4d3-4a658f065cad/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/cluster_a4adec14-55dc-2f49-b4d3-4a658f065cad/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T08:54:52,768 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62639 2024-11-24T08:54:52,778 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed with version=8 2024-11-24T08:54:52,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/hbase-staging 2024-11-24T08:54:52,781 INFO [Time-limited test {}] client.ConnectionUtils(128): master/469387a2cdb6:0 server-side Connection retries=45 2024-11-24T08:54:52,781 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:52,781 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:52,782 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:54:52,782 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:52,782 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:54:52,782 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T08:54:52,782 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:54:52,783 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44433 2024-11-24T08:54:52,785 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44433 connecting to ZooKeeper ensemble=127.0.0.1:62639 2024-11-24T08:54:52,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:444330x0, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:54:52,789 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44433-0x10070f0e2cc0000 connected 2024-11-24T08:54:52,803 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:52,803 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:52,805 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:52,805 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed, hbase.cluster.distributed=false 2024-11-24T08:54:52,808 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:54:52,809 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44433 2024-11-24T08:54:52,809 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44433 2024-11-24T08:54:52,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44433 2024-11-24T08:54:52,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44433 2024-11-24T08:54:52,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44433 2024-11-24T08:54:52,831 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/469387a2cdb6:0 server-side Connection retries=45 2024-11-24T08:54:52,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:52,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:52,831 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:54:52,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:52,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:54:52,831 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T08:54:52,831 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:54:52,832 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45065 2024-11-24T08:54:52,834 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45065 connecting to ZooKeeper ensemble=127.0.0.1:62639 2024-11-24T08:54:52,834 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:52,834 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:52,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:450650x0, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:54:52,837 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:450650x0, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:52,837 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45065-0x10070f0e2cc0001 connected 2024-11-24T08:54:52,838 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T08:54:52,838 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.cache 2024-11-24T08:54:52,839 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T08:54:52,839 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T08:54:52,839 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T08:54:52,839 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T08:54:52,839 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T08:54:52,839 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T08:54:52,840 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T08:54:52,840 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T08:54:52,840 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.persistence is complete. 2024-11-24T08:54:52,840 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T08:54:52,842 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:54:52,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45065 2024-11-24T08:54:52,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45065 2024-11-24T08:54:52,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45065 2024-11-24T08:54:52,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45065 2024-11-24T08:54:52,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45065 2024-11-24T08:54:52,860 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/469387a2cdb6:0 server-side Connection retries=45 2024-11-24T08:54:52,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:52,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:52,860 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:54:52,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:52,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:54:52,860 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T08:54:52,860 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:54:52,861 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36449 2024-11-24T08:54:52,863 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36449 connecting to ZooKeeper ensemble=127.0.0.1:62639 2024-11-24T08:54:52,863 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:52,863 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:52,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:364490x0, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:54:52,865 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36449-0x10070f0e2cc0002 connected 2024-11-24T08:54:52,866 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:52,866 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T08:54:52,866 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.cache 2024-11-24T08:54:52,866 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T08:54:52,867 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T08:54:52,867 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T08:54:52,867 INFO [Thread-122 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T08:54:52,867 WARN [Thread-122 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T08:54:52,867 INFO [Thread-122 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T08:54:52,867 INFO [Thread-122 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.persistence is complete. 2024-11-24T08:54:52,869 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T08:54:52,869 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T08:54:52,870 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T08:54:52,871 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:54:52,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36449 2024-11-24T08:54:52,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36449 2024-11-24T08:54:52,873 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36449 2024-11-24T08:54:52,873 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36449 2024-11-24T08:54:52,877 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36449 2024-11-24T08:54:52,889 DEBUG [M:0;469387a2cdb6:44433 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;469387a2cdb6:44433 2024-11-24T08:54:52,889 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/469387a2cdb6,44433,1732438492780 2024-11-24T08:54:52,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:52,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:52,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:52,891 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/469387a2cdb6,44433,1732438492780 2024-11-24T08:54:52,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T08:54:52,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T08:54:52,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:52,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:52,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:52,894 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T08:54:52,895 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/469387a2cdb6,44433,1732438492780 from backup master directory 2024-11-24T08:54:52,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/469387a2cdb6,44433,1732438492780 2024-11-24T08:54:52,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:52,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:52,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:54:52,896 WARN [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:54:52,896 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=469387a2cdb6,44433,1732438492780 2024-11-24T08:54:52,897 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/hbase.id] with ID: a2bd4dc8-0de3-4942-afa6-10050b23e3fc 2024-11-24T08:54:52,898 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/.tmp/hbase.id 2024-11-24T08:54:52,903 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/hbase.id] 2024-11-24T08:54:52,933 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:52,933 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T08:54:52,935 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-24T08:54:52,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:52,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:52,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:52,942 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:54:52,943 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T08:54:52,943 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:54:52,962 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store 2024-11-24T08:54:52,970 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:52,970 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T08:54:52,970 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:52,970 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:52,970 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T08:54:52,971 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:52,971 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:52,971 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732438492970Disabling compacts and flushes for region at 1732438492970Disabling writes for close at 1732438492970Writing region close event to WAL at 1732438492971 (+1 ms)Closed at 1732438492971 2024-11-24T08:54:52,971 WARN [master/469387a2cdb6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/.initializing 2024-11-24T08:54:52,971 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/WALs/469387a2cdb6,44433,1732438492780 2024-11-24T08:54:52,978 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:54:52,981 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=469387a2cdb6%2C44433%2C1732438492780, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/WALs/469387a2cdb6,44433,1732438492780, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/oldWALs, maxLogs=10 2024-11-24T08:54:52,997 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/WALs/469387a2cdb6,44433,1732438492780/469387a2cdb6%2C44433%2C1732438492780.1732438492982 2024-11-24T08:54:52,998 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T08:54:52,998 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:54:52,998 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:52,998 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:52,998 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:52,999 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:53,003 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T08:54:53,004 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,004 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:53,004 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:53,008 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T08:54:53,009 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,009 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:54:53,009 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:53,012 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T08:54:53,012 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,012 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:54:53,013 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:53,016 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T08:54:53,016 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,016 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:54:53,016 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:53,017 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:53,017 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:53,017 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:53,017 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:53,017 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T08:54:53,018 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:54:53,023 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:54:53,023 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68078350, jitterRate=0.014446467161178589}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T08:54:53,024 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732438492999Initializing all the Stores at 1732438492999Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438492999Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438492999Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438492999Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438492999Cleaning up temporary data from old regions at 1732438493017 (+18 ms)Region opened successfully at 1732438493024 (+7 ms) 2024-11-24T08:54:53,025 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T08:54:53,028 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@537d7161, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=469387a2cdb6/172.17.0.2:0 2024-11-24T08:54:53,030 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T08:54:53,030 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T08:54:53,030 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T08:54:53,030 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T08:54:53,030 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T08:54:53,031 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-24T08:54:53,031 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T08:54:53,033 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T08:54:53,034 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T08:54:53,035 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T08:54:53,036 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T08:54:53,036 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T08:54:53,037 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T08:54:53,037 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T08:54:53,038 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T08:54:53,039 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T08:54:53,040 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T08:54:53,041 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T08:54:53,043 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T08:54:53,044 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T08:54:53,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:53,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:53,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:53,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,046 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=469387a2cdb6,44433,1732438492780, sessionid=0x10070f0e2cc0000, setting cluster-up flag (Was=false) 2024-11-24T08:54:53,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,052 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T08:54:53,054 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=469387a2cdb6,44433,1732438492780 2024-11-24T08:54:53,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,059 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T08:54:53,061 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=469387a2cdb6,44433,1732438492780 2024-11-24T08:54:53,061 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T08:54:53,063 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T08:54:53,063 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T08:54:53,063 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T08:54:53,063 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 469387a2cdb6,44433,1732438492780 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T08:54:53,065 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/469387a2cdb6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:54:53,065 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/469387a2cdb6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:54:53,065 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/469387a2cdb6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:54:53,065 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/469387a2cdb6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:54:53,065 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/469387a2cdb6:0, corePoolSize=10, maxPoolSize=10 2024-11-24T08:54:53,066 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,066 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/469387a2cdb6:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:54:53,066 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,066 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732438523066 2024-11-24T08:54:53,067 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T08:54:53,067 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T08:54:53,067 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T08:54:53,067 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T08:54:53,067 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T08:54:53,067 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T08:54:53,067 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,067 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:54:53,067 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T08:54:53,067 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T08:54:53,067 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T08:54:53,068 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T08:54:53,068 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,068 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T08:54:53,069 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T08:54:53,069 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T08:54:53,070 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.large.0-1732438493069,5,FailOnTimeoutGroup] 2024-11-24T08:54:53,073 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.small.0-1732438493070,5,FailOnTimeoutGroup] 2024-11-24T08:54:53,073 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,073 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T08:54:53,073 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,073 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,079 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(746): ClusterId : a2bd4dc8-0de3-4942-afa6-10050b23e3fc 2024-11-24T08:54:53,079 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(746): ClusterId : a2bd4dc8-0de3-4942-afa6-10050b23e3fc 2024-11-24T08:54:53,079 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T08:54:53,079 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T08:54:53,082 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T08:54:53,082 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T08:54:53,082 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T08:54:53,082 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T08:54:53,084 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T08:54:53,084 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T08:54:53,085 DEBUG [RS:0;469387a2cdb6:45065 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55e86bc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=469387a2cdb6/172.17.0.2:0 2024-11-24T08:54:53,085 DEBUG [RS:1;469387a2cdb6:36449 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@356b1851, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=469387a2cdb6/172.17.0.2:0 2024-11-24T08:54:53,088 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T08:54:53,088 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed 2024-11-24T08:54:53,098 DEBUG [RS:0;469387a2cdb6:45065 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;469387a2cdb6:45065 2024-11-24T08:54:53,098 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T08:54:53,098 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T08:54:53,098 DEBUG [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T08:54:53,099 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(2659): reportForDuty to master=469387a2cdb6,44433,1732438492780 with port=45065, startcode=1732438492830 2024-11-24T08:54:53,099 DEBUG [RS:0;469387a2cdb6:45065 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T08:54:53,101 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:53,102 DEBUG [RS:1;469387a2cdb6:36449 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;469387a2cdb6:36449 2024-11-24T08:54:53,102 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T08:54:53,102 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T08:54:53,102 DEBUG [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T08:54:53,102 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41731, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T08:54:53,103 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(2659): reportForDuty to master=469387a2cdb6,44433,1732438492780 with port=36449, startcode=1732438492860 2024-11-24T08:54:53,103 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44433 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 469387a2cdb6,45065,1732438492830 2024-11-24T08:54:53,103 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44433 {}] master.ServerManager(517): Registering regionserver=469387a2cdb6,45065,1732438492830 2024-11-24T08:54:53,103 DEBUG [RS:1;469387a2cdb6:36449 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T08:54:53,105 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T08:54:53,105 DEBUG [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed 2024-11-24T08:54:53,105 DEBUG [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T08:54:53,105 DEBUG [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T08:54:53,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:54:53,109 DEBUG [RS:0;469387a2cdb6:45065 {}] zookeeper.ZKUtil(111): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/469387a2cdb6,45065,1732438492830 2024-11-24T08:54:53,109 WARN [RS:0;469387a2cdb6:45065 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:54:53,109 INFO [RS:0;469387a2cdb6:45065 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:54:53,109 DEBUG [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,45065,1732438492830 2024-11-24T08:54:53,116 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [469387a2cdb6,45065,1732438492830] 2024-11-24T08:54:53,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T08:54:53,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:53,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T08:54:53,118 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T08:54:53,118 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36101, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T08:54:53,118 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44433 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 469387a2cdb6,36449,1732438492860 2024-11-24T08:54:53,119 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44433 {}] master.ServerManager(517): Registering regionserver=469387a2cdb6,36449,1732438492860 2024-11-24T08:54:53,120 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T08:54:53,120 INFO [RS:0;469387a2cdb6:45065 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T08:54:53,120 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,120 DEBUG [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed 2024-11-24T08:54:53,121 DEBUG [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T08:54:53,121 DEBUG [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T08:54:53,122 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T08:54:53,122 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T08:54:53,122 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T08:54:53,122 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,122 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,123 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:53,123 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T08:54:53,123 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,123 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,123 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,123 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/469387a2cdb6:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:54:53,123 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,124 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,124 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,124 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,124 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,124 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,124 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/469387a2cdb6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:54:53,124 DEBUG [RS:0;469387a2cdb6:45065 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/469387a2cdb6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:54:53,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:54:53,125 DEBUG [RS:1;469387a2cdb6:36449 {}] zookeeper.ZKUtil(111): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/469387a2cdb6,36449,1732438492860 2024-11-24T08:54:53,125 WARN [RS:1;469387a2cdb6:36449 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:54:53,125 INFO [RS:1;469387a2cdb6:36449 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:54:53,126 DEBUG [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,36449,1732438492860 2024-11-24T08:54:53,126 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [469387a2cdb6,36449,1732438492860] 2024-11-24T08:54:53,130 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T08:54:53,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:53,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T08:54:53,133 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,133 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,133 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,133 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,133 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,133 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,45065,1732438492830-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:54:53,134 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T08:54:53,134 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,135 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T08:54:53,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:53,135 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T08:54:53,136 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740 2024-11-24T08:54:53,136 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740 2024-11-24T08:54:53,136 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T08:54:53,136 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T08:54:53,137 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T08:54:53,137 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T08:54:53,140 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T08:54:53,141 INFO [RS:1;469387a2cdb6:36449 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T08:54:53,141 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,141 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T08:54:53,142 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T08:54:53,142 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,142 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,143 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,143 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,143 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,143 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,143 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/469387a2cdb6:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:54:53,143 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,143 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,143 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,143 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,143 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,144 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:53,144 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/469387a2cdb6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:54:53,144 DEBUG [RS:1;469387a2cdb6:36449 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/469387a2cdb6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:54:53,151 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,151 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,151 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:54:53,151 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,151 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,151 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,151 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,36449,1732438492860-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:54:53,152 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62647561, jitterRate=-0.06647859513759613}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T08:54:53,152 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T08:54:53,152 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,45065,1732438492830-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,152 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,152 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732438493102Initializing all the Stores at 1732438493102Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438493102Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438493105 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438493105Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438493105Cleaning up temporary data from old regions at 1732438493136 (+31 ms)Region opened successfully at 1732438493152 (+16 ms) 2024-11-24T08:54:53,152 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.Replication(171): 469387a2cdb6,45065,1732438492830 started 2024-11-24T08:54:53,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T08:54:53,153 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T08:54:53,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T08:54:53,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T08:54:53,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T08:54:53,156 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T08:54:53,157 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732438493153Disabling compacts and flushes for region at 1732438493153Disabling writes for close at 1732438493153Writing region close event to WAL at 1732438493156 (+3 ms)Closed at 1732438493156 2024-11-24T08:54:53,158 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:54:53,158 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T08:54:53,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T08:54:53,159 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T08:54:53,160 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T08:54:53,169 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T08:54:53,169 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,36449,1732438492860-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,169 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,169 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(1482): Serving as 469387a2cdb6,45065,1732438492830, RpcServer on 469387a2cdb6/172.17.0.2:45065, sessionid=0x10070f0e2cc0001 2024-11-24T08:54:53,169 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,169 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.Replication(171): 469387a2cdb6,36449,1732438492860 started 2024-11-24T08:54:53,169 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T08:54:53,169 DEBUG [RS:0;469387a2cdb6:45065 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 469387a2cdb6,45065,1732438492830 2024-11-24T08:54:53,169 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '469387a2cdb6,45065,1732438492830' 2024-11-24T08:54:53,169 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T08:54:53,170 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T08:54:53,171 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T08:54:53,171 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T08:54:53,171 DEBUG [RS:0;469387a2cdb6:45065 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 469387a2cdb6,45065,1732438492830 2024-11-24T08:54:53,171 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '469387a2cdb6,45065,1732438492830' 2024-11-24T08:54:53,171 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T08:54:53,171 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T08:54:53,172 DEBUG [RS:0;469387a2cdb6:45065 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T08:54:53,172 INFO [RS:0;469387a2cdb6:45065 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T08:54:53,172 INFO [RS:0;469387a2cdb6:45065 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T08:54:53,185 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,185 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(1482): Serving as 469387a2cdb6,36449,1732438492860, RpcServer on 469387a2cdb6/172.17.0.2:36449, sessionid=0x10070f0e2cc0002 2024-11-24T08:54:53,185 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T08:54:53,185 DEBUG [RS:1;469387a2cdb6:36449 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 469387a2cdb6,36449,1732438492860 2024-11-24T08:54:53,185 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '469387a2cdb6,36449,1732438492860' 2024-11-24T08:54:53,185 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T08:54:53,186 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T08:54:53,187 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T08:54:53,187 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T08:54:53,187 DEBUG [RS:1;469387a2cdb6:36449 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 469387a2cdb6,36449,1732438492860 2024-11-24T08:54:53,187 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '469387a2cdb6,36449,1732438492860' 2024-11-24T08:54:53,187 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T08:54:53,188 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T08:54:53,188 DEBUG [RS:1;469387a2cdb6:36449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T08:54:53,188 INFO [RS:1;469387a2cdb6:36449 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T08:54:53,188 INFO [RS:1;469387a2cdb6:36449 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T08:54:53,274 INFO [RS:0;469387a2cdb6:45065 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:54:53,276 INFO [RS:0;469387a2cdb6:45065 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=469387a2cdb6%2C45065%2C1732438492830, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,45065,1732438492830, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs, maxLogs=55 2024-11-24T08:54:53,289 INFO [RS:1;469387a2cdb6:36449 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:54:53,290 INFO [RS:1;469387a2cdb6:36449 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=469387a2cdb6%2C36449%2C1732438492860, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,36449,1732438492860, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs, maxLogs=55 2024-11-24T08:54:53,300 INFO [RS:0;469387a2cdb6:45065 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,45065,1732438492830/469387a2cdb6%2C45065%2C1732438492830.1732438493280 2024-11-24T08:54:53,300 DEBUG [RS:0;469387a2cdb6:45065 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T08:54:53,310 DEBUG [469387a2cdb6:44433 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T08:54:53,311 INFO [RS:1;469387a2cdb6:36449 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,36449,1732438492860/469387a2cdb6%2C36449%2C1732438492860.1732438493291 2024-11-24T08:54:53,311 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=469387a2cdb6,45065,1732438492830 2024-11-24T08:54:53,312 DEBUG [RS:1;469387a2cdb6:36449 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T08:54:53,312 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 469387a2cdb6,45065,1732438492830, state=OPENING 2024-11-24T08:54:53,313 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T08:54:53,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:53,317 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:53,317 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:53,317 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:53,318 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T08:54:53,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=469387a2cdb6,45065,1732438492830}] 2024-11-24T08:54:53,471 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T08:54:53,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43965, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T08:54:53,480 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T08:54:53,480 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:54:53,481 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T08:54:53,482 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=469387a2cdb6%2C45065%2C1732438492830.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,45065,1732438492830, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs, maxLogs=55 2024-11-24T08:54:53,502 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,45065,1732438492830/469387a2cdb6%2C45065%2C1732438492830.meta.1732438493483.meta 2024-11-24T08:54:53,503 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T08:54:53,503 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:54:53,503 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T08:54:53,503 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T08:54:53,503 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T08:54:53,504 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T08:54:53,504 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:53,504 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T08:54:53,504 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T08:54:53,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T08:54:53,506 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T08:54:53,506 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:53,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T08:54:53,507 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T08:54:53,507 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:53,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T08:54:53,508 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T08:54:53,508 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,508 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:53,508 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T08:54:53,509 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T08:54:53,509 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:54:53,509 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T08:54:53,510 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740 2024-11-24T08:54:53,510 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740 2024-11-24T08:54:53,510 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T08:54:53,510 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T08:54:53,511 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T08:54:53,511 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T08:54:53,512 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59986596, jitterRate=-0.10613006353378296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T08:54:53,512 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T08:54:53,512 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732438493504Writing region info on filesystem at 1732438493504Initializing all the Stores at 1732438493504Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438493504Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438493505 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438493505Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732438493505Cleaning up temporary data from old regions at 1732438493510 (+5 ms)Running coprocessor post-open hooks at 1732438493512 (+2 ms)Region opened successfully at 1732438493512 2024-11-24T08:54:53,513 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732438493471 2024-11-24T08:54:53,516 DEBUG [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T08:54:53,516 INFO [RS_OPEN_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T08:54:53,516 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=469387a2cdb6,45065,1732438492830 2024-11-24T08:54:53,517 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 469387a2cdb6,45065,1732438492830, state=OPEN 2024-11-24T08:54:53,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:54:53,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:54:53,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:54:53,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:53,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:53,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:54:53,518 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=469387a2cdb6,45065,1732438492830 2024-11-24T08:54:53,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T08:54:53,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=469387a2cdb6,45065,1732438492830 in 201 msec 2024-11-24T08:54:53,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T08:54:53,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 363 msec 2024-11-24T08:54:53,523 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:54:53,523 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T08:54:53,526 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:54:53,526 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=469387a2cdb6,45065,1732438492830, seqNum=-1] 2024-11-24T08:54:53,526 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:54:53,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60375, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:54:53,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 470 msec 2024-11-24T08:54:53,533 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732438493533, completionTime=-1 2024-11-24T08:54:53,533 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T08:54:53,533 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T08:54:53,535 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T08:54:53,536 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732438553535 2024-11-24T08:54:53,536 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732438613536 2024-11-24T08:54:53,536 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-24T08:54:53,536 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,44433,1732438492780-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,536 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,44433,1732438492780-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,536 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,44433,1732438492780-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,536 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-469387a2cdb6:44433, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,536 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,537 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,539 DEBUG [master/469387a2cdb6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T08:54:53,541 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.645sec 2024-11-24T08:54:53,541 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T08:54:53,541 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T08:54:53,542 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T08:54:53,542 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T08:54:53,542 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T08:54:53,542 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,44433,1732438492780-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:54:53,542 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,44433,1732438492780-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T08:54:53,543 DEBUG [master/469387a2cdb6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T08:54:53,543 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T08:54:53,543 INFO [master/469387a2cdb6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,44433,1732438492780-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:53,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d25b74f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:54:53,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 469387a2cdb6,44433,-1 for getting cluster id 2024-11-24T08:54:53,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:54:53,583 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a2bd4dc8-0de3-4942-afa6-10050b23e3fc' 2024-11-24T08:54:53,584 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:54:53,584 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a2bd4dc8-0de3-4942-afa6-10050b23e3fc" 2024-11-24T08:54:53,585 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc4b196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:54:53,585 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [469387a2cdb6,44433,-1] 2024-11-24T08:54:53,586 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:54:53,586 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:53,589 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:54:53,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5858d96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:54:53,592 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:54:53,594 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=469387a2cdb6,45065,1732438492830, seqNum=-1] 2024-11-24T08:54:53,594 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:54:53,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:54:53,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=469387a2cdb6,44433,1732438492780 2024-11-24T08:54:53,598 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T08:54:53,600 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 469387a2cdb6,44433,1732438492780 2024-11-24T08:54:53,600 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4fb3ba58 2024-11-24T08:54:53,600 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T08:54:53,602 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T08:54:53,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732438493598', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:54:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732438493598 2024-11-24T08:54:53,604 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732438493598 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T08:54:53,604 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732438493598" procId is: 4 2024-11-24T08:54:53,605 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732438493598 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T08:54:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:54:53,615 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732438493598-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5dc0d1664f37315aa7ff0a71dcd67668, NAME => 'testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732438493598', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed 2024-11-24T08:54:53,621 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732438493598-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:53,622 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732438493598-pool-0 {}] regionserver.HRegion(1722): Closing 5dc0d1664f37315aa7ff0a71dcd67668, disabling compactions & flushes 2024-11-24T08:54:53,622 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732438493598-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:53,622 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732438493598-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:53,622 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732438493598-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. after waiting 0 ms 2024-11-24T08:54:53,622 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732438493598-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:53,622 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732438493598-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:53,622 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732438493598-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5dc0d1664f37315aa7ff0a71dcd67668: Waiting for close lock at 1732438493622Disabling compacts and flushes for region at 1732438493622Disabling writes for close at 1732438493622Writing region close event to WAL at 1732438493622Closed at 1732438493622 2024-11-24T08:54:53,623 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732438493598 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T08:54:53,624 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732438493623"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732438493623"}]},"ts":"1732438493623"} 2024-11-24T08:54:53,626 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T08:54:53,627 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732438493598 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T08:54:53,627 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732438493598","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732438493627"}]},"ts":"1732438493627"} 2024-11-24T08:54:53,629 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732438493598, state=ENABLING in hbase:meta 2024-11-24T08:54:53,630 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {469387a2cdb6=0} racks are {/default-rack=0} 2024-11-24T08:54:53,630 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T08:54:53,630 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T08:54:53,630 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T08:54:53,630 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T08:54:53,630 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T08:54:53,631 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T08:54:53,631 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T08:54:53,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732438493598, region=5dc0d1664f37315aa7ff0a71dcd67668, ASSIGN}] 2024-11-24T08:54:53,632 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732438493598, region=5dc0d1664f37315aa7ff0a71dcd67668, ASSIGN 2024-11-24T08:54:53,633 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732438493598, region=5dc0d1664f37315aa7ff0a71dcd67668, ASSIGN; state=OFFLINE, location=469387a2cdb6,36449,1732438492860; forceNewPlan=false, retain=false 2024-11-24T08:54:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:54:53,784 INFO [469387a2cdb6:44433 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T08:54:53,784 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5dc0d1664f37315aa7ff0a71dcd67668, regionState=OPENING, regionLocation=469387a2cdb6,36449,1732438492860 2024-11-24T08:54:53,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732438493598, region=5dc0d1664f37315aa7ff0a71dcd67668, ASSIGN because future has completed 2024-11-24T08:54:53,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dc0d1664f37315aa7ff0a71dcd67668, server=469387a2cdb6,36449,1732438492860}] 2024-11-24T08:54:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:54:53,945 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T08:54:53,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45629, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T08:54:53,949 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:53,950 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5dc0d1664f37315aa7ff0a71dcd67668, NAME => 'testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:54:53,950 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732438493598 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:53,950 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:53,951 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:53,951 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:53,952 INFO [StoreOpener-5dc0d1664f37315aa7ff0a71dcd67668-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:53,959 INFO [StoreOpener-5dc0d1664f37315aa7ff0a71dcd67668-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dc0d1664f37315aa7ff0a71dcd67668 columnFamilyName family 2024-11-24T08:54:53,959 DEBUG [StoreOpener-5dc0d1664f37315aa7ff0a71dcd67668-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:53,959 INFO [StoreOpener-5dc0d1664f37315aa7ff0a71dcd67668-1 {}] regionserver.HStore(327): Store=5dc0d1664f37315aa7ff0a71dcd67668/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:54:53,959 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:53,961 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:53,961 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:53,961 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:53,961 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:53,962 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:53,967 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:54:53,967 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5dc0d1664f37315aa7ff0a71dcd67668; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69881132, jitterRate=0.041310012340545654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:54:53,967 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:53,968 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5dc0d1664f37315aa7ff0a71dcd67668: Running coprocessor pre-open hook at 1732438493951Writing region info on filesystem at 1732438493951Initializing all the Stores at 1732438493951Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438493951Cleaning up temporary data from old regions at 1732438493961 (+10 ms)Running coprocessor post-open hooks at 1732438493967 (+6 ms)Region opened successfully at 1732438493968 (+1 ms) 2024-11-24T08:54:53,969 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668., pid=6, masterSystemTime=1732438493944 2024-11-24T08:54:53,971 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:53,971 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:53,971 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5dc0d1664f37315aa7ff0a71dcd67668, regionState=OPEN, openSeqNum=2, regionLocation=469387a2cdb6,36449,1732438492860 2024-11-24T08:54:53,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dc0d1664f37315aa7ff0a71dcd67668, server=469387a2cdb6,36449,1732438492860 because future has completed 2024-11-24T08:54:53,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T08:54:53,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5dc0d1664f37315aa7ff0a71dcd67668, server=469387a2cdb6,36449,1732438492860 in 185 msec 2024-11-24T08:54:53,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T08:54:53,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732438493598, region=5dc0d1664f37315aa7ff0a71dcd67668, ASSIGN in 347 msec 2024-11-24T08:54:53,980 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732438493598 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T08:54:53,981 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732438493598","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732438493980"}]},"ts":"1732438493980"} 2024-11-24T08:54:53,983 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732438493598, state=ENABLED in hbase:meta 2024-11-24T08:54:53,984 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732438493598 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T08:54:53,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732438493598 in 381 msec 2024-11-24T08:54:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:54:54,235 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732438493598 completed 2024-11-24T08:54:54,235 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732438493598 get assigned. Timeout = 60000ms 2024-11-24T08:54:54,236 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T08:54:54,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732438493598 assigned to meta. Checking AM states. 2024-11-24T08:54:54,244 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T08:54:54,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732438493598 assigned. 2024-11-24T08:54:54,251 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732438493598', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668., hostname=469387a2cdb6,36449,1732438492860, seqNum=2] 2024-11-24T08:54:54,252 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:54:54,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:54:54,258 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5dc0d1664f37315aa7ff0a71dcd67668 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T08:54:54,266 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8643f4c434754e938d50f24bc215eb2a 2024-11-24T08:54:54,277 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/.tmp/family/8643f4c434754e938d50f24bc215eb2a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T08:54:54,278 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/.tmp/family/8643f4c434754e938d50f24bc215eb2a 2024-11-24T08:54:54,281 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/.tmp/family/8643f4c434754e938d50f24bc215eb2a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/family/8643f4c434754e938d50f24bc215eb2a 2024-11-24T08:54:54,282 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/family/8643f4c434754e938d50f24bc215eb2a, delay=948 ms 2024-11-24T08:54:54,283 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/family/8643f4c434754e938d50f24bc215eb2a, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T08:54:54,284 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 5dc0d1664f37315aa7ff0a71dcd67668 in 27ms, sequenceid=6, compaction requested=false 2024-11-24T08:54:54,284 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732438493598' 2024-11-24T08:54:54,284 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5dc0d1664f37315aa7ff0a71dcd67668: 2024-11-24T08:54:55,077 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T08:54:55,231 DEBUG [hfile-prefetch-1732438492247 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8643f4c434754e938d50f24bc215eb2a_0. 2024-11-24T08:54:55,233 DEBUG [hfile-prefetch-1732438492247 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8643f4c434754e938d50f24bc215eb2a_117. 2024-11-24T08:54:55,233 DEBUG [hfile-prefetch-1732438492247 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/family/8643f4c434754e938d50f24bc215eb2a, with total blocks 2, and data blocks 1 2024-11-24T08:54:55,234 DEBUG [hfile-prefetch-1732438492247 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T08:54:55,234 DEBUG [hfile-prefetch-1732438492247 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/family/8643f4c434754e938d50f24bc215eb2a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T08:54:55,234 DEBUG [hfile-prefetch-1732438492247 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/family/8643f4c434754e938d50f24bc215eb2a. Will sleep 100ms and try the verification again. 2024-11-24T08:54:55,241 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T08:54:55,241 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T08:54:55,242 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T08:54:55,242 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T08:54:55,246 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T08:54:55,246 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732438495241 2024-11-24T08:54:55,246 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T08:54:55,285 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;469387a2cdb6:36449,5,FailOnTimeoutGroup] 2024-11-24T08:54:55,285 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '469387a2cdb6,36449,1732438492860' ***** 2024-11-24T08:54:55,285 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-24T08:54:55,286 INFO [RS:1;469387a2cdb6:36449 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T08:54:55,286 INFO [RS:1;469387a2cdb6:36449 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.persistence 2024-11-24T08:54:55,286 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T08:54:55,286 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T08:54:55,286 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T08:54:55,286 INFO [RS:1;469387a2cdb6:36449 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T08:54:55,286 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T08:54:55,296 DEBUG [RS:1;469387a2cdb6:36449 {}] bucket.BucketCache(1487): Thread RS:1;469387a2cdb6:36449 started persisting bucket cache to file 2024-11-24T08:54:55,296 DEBUG [RS:1;469387a2cdb6:36449 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T08:54:55,296 DEBUG [RS:1;469387a2cdb6:36449 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T08:54:55,299 DEBUG [RS:1;469387a2cdb6:36449 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-24T08:54:55,299 DEBUG [RS:1;469387a2cdb6:36449 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732438495296 2024-11-24T08:54:55,299 DEBUG [RS:1;469387a2cdb6:36449 {}] bucket.BucketCache(1509): Thread RS:1;469387a2cdb6:36449 finished persisting bucket cache to file, renaming 2024-11-24T08:54:55,300 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T08:54:55,300 INFO [RS:1;469387a2cdb6:36449 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T08:54:55,300 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T08:54:55,300 INFO [RS:1;469387a2cdb6:36449 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T08:54:55,300 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(3091): Received CLOSE for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,300 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(959): stopping server 469387a2cdb6,36449,1732438492860 2024-11-24T08:54:55,300 INFO [RS:1;469387a2cdb6:36449 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:54:55,300 INFO [RS:1;469387a2cdb6:36449 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;469387a2cdb6:36449. 2024-11-24T08:54:55,300 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T08:54:55,300 DEBUG [RS:1;469387a2cdb6:36449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:54:55,300 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5dc0d1664f37315aa7ff0a71dcd67668, disabling compactions & flushes 2024-11-24T08:54:55,300 DEBUG [RS:1;469387a2cdb6:36449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:55,300 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:55,301 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:55,301 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T08:54:55,301 DEBUG [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(1325): Online Regions={5dc0d1664f37315aa7ff0a71dcd67668=testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668.} 2024-11-24T08:54:55,301 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. after waiting 0 ms 2024-11-24T08:54:55,301 DEBUG [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(1351): Waiting on 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,301 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:55,301 DEBUG [StoreFileCloser-5dc0d1664f37315aa7ff0a71dcd67668-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/family/8643f4c434754e938d50f24bc215eb2a 2024-11-24T08:54:55,301 WARN [hfile-prefetch-1732438492247 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/family/8643f4c434754e938d50f24bc215eb2a, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-24T08:54:55,302 DEBUG [hfile-prefetch-1732438492247 {}] hfile.PrefetchExecutor(121): Prefetch completed for 8643f4c434754e938d50f24bc215eb2a 2024-11-24T08:54:55,305 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T08:54:55,305 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:55,306 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5dc0d1664f37315aa7ff0a71dcd67668: Waiting for close lock at 1732438495300Running coprocessor pre-close hooks at 1732438495300Disabling compacts and flushes for region at 1732438495300Disabling writes for close at 1732438495301 (+1 ms)Writing region close event to WAL at 1732438495302 (+1 ms)Running coprocessor post-close hooks at 1732438495305 (+3 ms)Closed at 1732438495305 2024-11-24T08:54:55,306 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:55,356 INFO [regionserver/469387a2cdb6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:54:55,501 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(976): stopping server 469387a2cdb6,36449,1732438492860; all regions closed. 2024-11-24T08:54:55,508 DEBUG [RS:1;469387a2cdb6:36449 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs 2024-11-24T08:54:55,508 INFO [RS:1;469387a2cdb6:36449 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 469387a2cdb6%2C36449%2C1732438492860:(num 1732438493291) 2024-11-24T08:54:55,508 DEBUG [RS:1;469387a2cdb6:36449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:55,508 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:54:55,509 INFO [RS:1;469387a2cdb6:36449 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:54:55,510 INFO [RS:1;469387a2cdb6:36449 {}] hbase.ChoreService(370): Chore service for: regionserver/469387a2cdb6:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T08:54:55,510 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T08:54:55,511 INFO [regionserver/469387a2cdb6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:54:55,511 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T08:54:55,511 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T08:54:55,511 INFO [RS:1;469387a2cdb6:36449 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:54:55,511 INFO [RS:1;469387a2cdb6:36449 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36449 2024-11-24T08:54:55,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/469387a2cdb6,36449,1732438492860 2024-11-24T08:54:55,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:54:55,516 INFO [RS:1;469387a2cdb6:36449 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:54:55,517 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [469387a2cdb6,36449,1732438492860] 2024-11-24T08:54:55,518 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/469387a2cdb6,36449,1732438492860 already deleted, retry=false 2024-11-24T08:54:55,518 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 469387a2cdb6,36449,1732438492860 on 469387a2cdb6,44433,1732438492780 2024-11-24T08:54:55,521 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 469387a2cdb6,36449,1732438492860, splitWal=true, meta=false 2024-11-24T08:54:55,523 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 469387a2cdb6,36449,1732438492860 (carryingMeta=false) 469387a2cdb6,36449,1732438492860/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7e73a501[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T08:54:55,523 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 469387a2cdb6,36449,1732438492860, splitWal=true, meta=false 2024-11-24T08:54:55,525 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 469387a2cdb6,36449,1732438492860 had 1 regions 2024-11-24T08:54:55,525 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 469387a2cdb6,36449,1732438492860, splitWal=true, meta=false, isMeta: false 2024-11-24T08:54:55,526 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,36449,1732438492860-splitting 2024-11-24T08:54:55,526 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,36449,1732438492860-splitting dir is empty, no logs to split. 2024-11-24T08:54:55,526 INFO [PEWorker-1 {}] master.SplitWALManager(105): 469387a2cdb6,36449,1732438492860 WAL count=0, meta=false 2024-11-24T08:54:55,528 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,36449,1732438492860-splitting dir is empty, no logs to split. 2024-11-24T08:54:55,528 INFO [PEWorker-1 {}] master.SplitWALManager(105): 469387a2cdb6,36449,1732438492860 WAL count=0, meta=false 2024-11-24T08:54:55,528 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 469387a2cdb6,36449,1732438492860 WAL splitting is done? wals=0, meta=false 2024-11-24T08:54:55,528 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 469387a2cdb6,36449,1732438492860 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,36449,1732438492860-splitting does not exist 2024-11-24T08:54:55,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732438493598, region=5dc0d1664f37315aa7ff0a71dcd67668, ASSIGN}] 2024-11-24T08:54:55,531 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732438493598, region=5dc0d1664f37315aa7ff0a71dcd67668, ASSIGN 2024-11-24T08:54:55,532 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732438493598, region=5dc0d1664f37315aa7ff0a71dcd67668, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T08:54:55,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:55,617 INFO [RS:1;469387a2cdb6:36449 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:54:55,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36449-0x10070f0e2cc0002, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:55,617 INFO [RS:1;469387a2cdb6:36449 {}] regionserver.HRegionServer(1031): Exiting; stopping=469387a2cdb6,36449,1732438492860; zookeeper connection closed. 2024-11-24T08:54:55,617 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@490f805c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@490f805c 2024-11-24T08:54:55,683 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5dc0d1664f37315aa7ff0a71dcd67668, regionState=OPENING, regionLocation=469387a2cdb6,45065,1732438492830 2024-11-24T08:54:55,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732438493598, region=5dc0d1664f37315aa7ff0a71dcd67668, ASSIGN because future has completed 2024-11-24T08:54:55,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dc0d1664f37315aa7ff0a71dcd67668, server=469387a2cdb6,45065,1732438492830}] 2024-11-24T08:54:55,805 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/469387a2cdb6:0 server-side Connection retries=45 2024-11-24T08:54:55,805 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:55,805 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:55,805 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:54:55,805 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:54:55,805 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:54:55,805 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T08:54:55,805 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:54:55,806 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40459 2024-11-24T08:54:55,807 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40459 connecting to ZooKeeper ensemble=127.0.0.1:62639 2024-11-24T08:54:55,808 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:55,808 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:55,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404590x0, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:54:55,811 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40459-0x10070f0e2cc0003 connected 2024-11-24T08:54:55,811 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40459-0x10070f0e2cc0003, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-24T08:54:55,811 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-24T08:54:55,812 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T08:54:55,812 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.cache 2024-11-24T08:54:55,812 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T08:54:55,812 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T08:54:55,812 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T08:54:55,813 INFO [Thread-202 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T08:54:55,813 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T08:54:55,814 INFO [Thread-202 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-24T08:54:55,815 INFO [Thread-202 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-24T08:54:55,816 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T08:54:55,817 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40459-0x10070f0e2cc0003, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T08:54:55,819 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40459-0x10070f0e2cc0003, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:54:55,820 DEBUG [Thread-202 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-24T08:54:55,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40459 2024-11-24T08:54:55,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40459 2024-11-24T08:54:55,820 INFO [Thread-202 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T08:54:55,821 INFO [Thread-202 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-24T08:54:55,821 INFO [Thread-202 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.persistence is complete. 2024-11-24T08:54:55,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40459 2024-11-24T08:54:55,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40459 2024-11-24T08:54:55,829 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40459 2024-11-24T08:54:55,830 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer(746): ClusterId : a2bd4dc8-0de3-4942-afa6-10050b23e3fc 2024-11-24T08:54:55,830 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T08:54:55,832 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T08:54:55,832 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T08:54:55,834 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T08:54:55,834 DEBUG [RS:2;469387a2cdb6:40459 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6c85b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=469387a2cdb6/172.17.0.2:0 2024-11-24T08:54:55,844 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:55,844 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 5dc0d1664f37315aa7ff0a71dcd67668, NAME => 'testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:54:55,845 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732438493598 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,845 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:54:55,845 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,845 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,846 INFO [StoreOpener-5dc0d1664f37315aa7ff0a71dcd67668-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,847 INFO [StoreOpener-5dc0d1664f37315aa7ff0a71dcd67668-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dc0d1664f37315aa7ff0a71dcd67668 columnFamilyName family 2024-11-24T08:54:55,847 DEBUG [StoreOpener-5dc0d1664f37315aa7ff0a71dcd67668-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:54:55,848 DEBUG [RS:2;469387a2cdb6:40459 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;469387a2cdb6:40459 2024-11-24T08:54:55,848 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T08:54:55,848 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T08:54:55,848 DEBUG [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T08:54:55,849 DEBUG [StoreFileOpener-5dc0d1664f37315aa7ff0a71dcd67668-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/family/8643f4c434754e938d50f24bc215eb2a, delay=954 ms 2024-11-24T08:54:55,849 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer(2659): reportForDuty to master=469387a2cdb6,44433,1732438492780 with port=40459, startcode=1732438495804 2024-11-24T08:54:55,849 DEBUG [RS:2;469387a2cdb6:40459 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T08:54:55,850 DEBUG [StoreOpener-5dc0d1664f37315aa7ff0a71dcd67668-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/family/8643f4c434754e938d50f24bc215eb2a 2024-11-24T08:54:55,850 INFO [StoreOpener-5dc0d1664f37315aa7ff0a71dcd67668-1 {}] regionserver.HStore(327): Store=5dc0d1664f37315aa7ff0a71dcd67668/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:54:55,850 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,851 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,851 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42059, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T08:54:55,852 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,852 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,852 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44433 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 469387a2cdb6,40459,1732438495804 2024-11-24T08:54:55,852 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,852 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44433 {}] master.ServerManager(517): Registering regionserver=469387a2cdb6,40459,1732438495804 2024-11-24T08:54:55,854 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,854 DEBUG [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed 2024-11-24T08:54:55,854 DEBUG [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T08:54:55,854 DEBUG [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T08:54:55,855 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 5dc0d1664f37315aa7ff0a71dcd67668; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61783440, jitterRate=-0.07935500144958496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:54:55,855 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:54:55,856 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 5dc0d1664f37315aa7ff0a71dcd67668: Running coprocessor pre-open hook at 1732438495846Writing region info on filesystem at 1732438495846Initializing all the Stores at 1732438495846Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732438495846Cleaning up temporary data from old regions at 1732438495852 (+6 ms)Running coprocessor post-open hooks at 1732438495855 (+3 ms)Region opened successfully at 1732438495856 (+1 ms) 2024-11-24T08:54:55,857 DEBUG [RS:2;469387a2cdb6:40459 {}] zookeeper.ZKUtil(111): regionserver:40459-0x10070f0e2cc0003, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/469387a2cdb6,40459,1732438495804 2024-11-24T08:54:55,857 WARN [RS:2;469387a2cdb6:40459 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:54:55,857 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [469387a2cdb6,40459,1732438495804] 2024-11-24T08:54:55,857 INFO [RS:2;469387a2cdb6:40459 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:54:55,857 DEBUG [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,40459,1732438495804 2024-11-24T08:54:55,857 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668., pid=9, masterSystemTime=1732438495842 2024-11-24T08:54:55,859 DEBUG [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:55,860 INFO [RS_OPEN_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:55,860 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5dc0d1664f37315aa7ff0a71dcd67668, regionState=OPEN, openSeqNum=10, regionLocation=469387a2cdb6,45065,1732438492830 2024-11-24T08:54:55,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dc0d1664f37315aa7ff0a71dcd67668, server=469387a2cdb6,45065,1732438492830 because future has completed 2024-11-24T08:54:55,864 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T08:54:55,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-24T08:54:55,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 5dc0d1664f37315aa7ff0a71dcd67668, server=469387a2cdb6,45065,1732438492830 in 175 msec 2024-11-24T08:54:55,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T08:54:55,868 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 469387a2cdb6,36449,1732438492860 after splitting done 2024-11-24T08:54:55,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732438493598, region=5dc0d1664f37315aa7ff0a71dcd67668, ASSIGN in 336 msec 2024-11-24T08:54:55,869 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T08:54:55,870 INFO [RS:2;469387a2cdb6:40459 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T08:54:55,870 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:55,871 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 469387a2cdb6,36449,1732438492860, splitWal=true, meta=false in 351 msec 2024-11-24T08:54:55,871 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T08:54:55,871 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T08:54:55,871 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:55,871 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:55,871 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:55,871 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:55,872 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:55,872 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:55,872 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/469387a2cdb6:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:54:55,872 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:55,872 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:55,872 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:55,872 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:55,872 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:55,872 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/469387a2cdb6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:54:55,872 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/469387a2cdb6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:54:55,872 DEBUG [RS:2;469387a2cdb6:40459 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/469387a2cdb6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:54:55,873 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:55,873 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:55,873 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:55,873 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:55,873 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:55,873 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,40459,1732438495804-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:54:55,889 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T08:54:55,889 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(168): Chore ScheduledChore name=469387a2cdb6,40459,1732438495804-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:55,889 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:55,889 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.Replication(171): 469387a2cdb6,40459,1732438495804 started 2024-11-24T08:54:55,905 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:54:55,905 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer(1482): Serving as 469387a2cdb6,40459,1732438495804, RpcServer on 469387a2cdb6/172.17.0.2:40459, sessionid=0x10070f0e2cc0003 2024-11-24T08:54:55,905 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T08:54:55,905 DEBUG [RS:2;469387a2cdb6:40459 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 469387a2cdb6,40459,1732438495804 2024-11-24T08:54:55,905 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '469387a2cdb6,40459,1732438495804' 2024-11-24T08:54:55,905 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T08:54:55,906 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T08:54:55,907 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T08:54:55,907 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T08:54:55,907 DEBUG [RS:2;469387a2cdb6:40459 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 469387a2cdb6,40459,1732438495804 2024-11-24T08:54:55,907 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '469387a2cdb6,40459,1732438495804' 2024-11-24T08:54:55,907 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T08:54:55,907 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T08:54:55,908 DEBUG [RS:2;469387a2cdb6:40459 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T08:54:55,908 INFO [RS:2;469387a2cdb6:40459 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T08:54:55,908 INFO [RS:2;469387a2cdb6:40459 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T08:54:56,009 INFO [RS:2;469387a2cdb6:40459 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:54:56,010 INFO [RS:2;469387a2cdb6:40459 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=469387a2cdb6%2C40459%2C1732438495804, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,40459,1732438495804, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs, maxLogs=55 2024-11-24T08:54:56,031 INFO [RS:2;469387a2cdb6:40459 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/WALs/469387a2cdb6,40459,1732438495804/469387a2cdb6%2C40459%2C1732438495804.1732438496012 2024-11-24T08:54:56,031 DEBUG [RS:2;469387a2cdb6:40459 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T08:54:56,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T08:54:56,405 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:54:56,406 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:54:56,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:56,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:56,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:54:56,406 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T08:54:56,406 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1393918527, stopped=false 2024-11-24T08:54:56,407 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=469387a2cdb6,44433,1732438492780 2024-11-24T08:54:56,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40459-0x10070f0e2cc0003, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:56,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:56,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T08:54:56,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:56,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40459-0x10070f0e2cc0003, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:56,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:56,408 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T08:54:56,409 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:54:56,409 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:54:56,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:56,409 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:56,410 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40459-0x10070f0e2cc0003, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:56,410 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '469387a2cdb6,45065,1732438492830' ***** 2024-11-24T08:54:56,410 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T08:54:56,410 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '469387a2cdb6,40459,1732438495804' ***** 2024-11-24T08:54:56,410 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:54:56,410 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T08:54:56,410 INFO [RS:0;469387a2cdb6:45065 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T08:54:56,410 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T08:54:56,410 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T08:54:56,410 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T08:54:56,411 INFO [RS:0;469387a2cdb6:45065 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.persistence 2024-11-24T08:54:56,411 INFO [RS:2;469387a2cdb6:40459 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T08:54:56,411 INFO [RS:0;469387a2cdb6:45065 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T08:54:56,411 INFO [RS:2;469387a2cdb6:40459 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/97a31dd2-1fe0-cfcd-2ac9-0ef43d254588/bucket.persistence 2024-11-24T08:54:56,411 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T08:54:56,411 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T08:54:56,411 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T08:54:56,411 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T08:54:56,412 INFO [RS:2;469387a2cdb6:40459 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T08:54:56,412 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T08:54:56,412 DEBUG [RS:2;469387a2cdb6:40459 {}] bucket.BucketCache(1487): Thread RS:2;469387a2cdb6:40459 started persisting bucket cache to file 2024-11-24T08:54:56,412 DEBUG [RS:2;469387a2cdb6:40459 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T08:54:56,413 DEBUG [RS:2;469387a2cdb6:40459 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T08:54:56,416 DEBUG [RS:2;469387a2cdb6:40459 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-24T08:54:56,416 DEBUG [RS:2;469387a2cdb6:40459 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732438496412 2024-11-24T08:54:56,417 DEBUG [RS:2;469387a2cdb6:40459 {}] bucket.BucketCache(1509): Thread RS:2;469387a2cdb6:40459 finished persisting bucket cache to file, renaming 2024-11-24T08:54:56,417 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T08:54:56,417 INFO [RS:2;469387a2cdb6:40459 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T08:54:56,417 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T08:54:56,417 INFO [RS:2;469387a2cdb6:40459 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T08:54:56,417 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer(959): stopping server 469387a2cdb6,40459,1732438495804 2024-11-24T08:54:56,417 INFO [RS:2;469387a2cdb6:40459 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:54:56,417 INFO [RS:2;469387a2cdb6:40459 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;469387a2cdb6:40459. 2024-11-24T08:54:56,417 DEBUG [RS:2;469387a2cdb6:40459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:54:56,417 DEBUG [RS:2;469387a2cdb6:40459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:56,417 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer(976): stopping server 469387a2cdb6,40459,1732438495804; all regions closed. 2024-11-24T08:54:56,419 DEBUG [RS:2;469387a2cdb6:40459 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs 2024-11-24T08:54:56,419 INFO [RS:2;469387a2cdb6:40459 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 469387a2cdb6%2C40459%2C1732438495804:(num 1732438496012) 2024-11-24T08:54:56,419 DEBUG [RS:2;469387a2cdb6:40459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:56,419 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:54:56,419 INFO [RS:2;469387a2cdb6:40459 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:54:56,419 INFO [RS:2;469387a2cdb6:40459 {}] hbase.ChoreService(370): Chore service for: regionserver/469387a2cdb6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T08:54:56,419 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T08:54:56,419 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T08:54:56,420 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T08:54:56,420 INFO [regionserver/469387a2cdb6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:54:56,420 INFO [RS:2;469387a2cdb6:40459 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:54:56,420 INFO [RS:2;469387a2cdb6:40459 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40459 2024-11-24T08:54:56,421 DEBUG [RS:0;469387a2cdb6:45065 {}] bucket.BucketCache(1487): Thread RS:0;469387a2cdb6:45065 started persisting bucket cache to file 2024-11-24T08:54:56,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40459-0x10070f0e2cc0003, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/469387a2cdb6,40459,1732438495804 2024-11-24T08:54:56,421 DEBUG [RS:0;469387a2cdb6:45065 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T08:54:56,421 INFO [RS:2;469387a2cdb6:40459 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:54:56,421 DEBUG [RS:0;469387a2cdb6:45065 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T08:54:56,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:54:56,422 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [469387a2cdb6,40459,1732438495804] 2024-11-24T08:54:56,423 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/469387a2cdb6,40459,1732438495804 already deleted, retry=false 2024-11-24T08:54:56,423 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 469387a2cdb6,40459,1732438495804 expired; onlineServers=1 2024-11-24T08:54:56,424 DEBUG [RS:0;469387a2cdb6:45065 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T08:54:56,424 DEBUG [RS:0;469387a2cdb6:45065 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732438496421 2024-11-24T08:54:56,424 DEBUG [RS:0;469387a2cdb6:45065 {}] bucket.BucketCache(1509): Thread RS:0;469387a2cdb6:45065 finished persisting bucket cache to file, renaming 2024-11-24T08:54:56,424 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T08:54:56,425 INFO [RS:0;469387a2cdb6:45065 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T08:54:56,425 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T08:54:56,425 INFO [RS:0;469387a2cdb6:45065 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T08:54:56,425 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(3091): Received CLOSE for 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:56,425 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(959): stopping server 469387a2cdb6,45065,1732438492830 2024-11-24T08:54:56,425 INFO [RS:0;469387a2cdb6:45065 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:54:56,425 INFO [RS:0;469387a2cdb6:45065 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;469387a2cdb6:45065. 2024-11-24T08:54:56,425 DEBUG [RS:0;469387a2cdb6:45065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:54:56,425 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T08:54:56,425 DEBUG [RS:0;469387a2cdb6:45065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:56,425 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5dc0d1664f37315aa7ff0a71dcd67668, disabling compactions & flushes 2024-11-24T08:54:56,425 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:56,425 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T08:54:56,425 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:56,425 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T08:54:56,425 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. after waiting 0 ms 2024-11-24T08:54:56,425 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T08:54:56,425 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:56,426 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T08:54:56,426 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T08:54:56,426 DEBUG [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(1325): Online Regions={5dc0d1664f37315aa7ff0a71dcd67668=testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668., 1588230740=hbase:meta,,1.1588230740} 2024-11-24T08:54:56,426 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T08:54:56,426 DEBUG [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5dc0d1664f37315aa7ff0a71dcd67668 2024-11-24T08:54:56,426 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T08:54:56,426 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T08:54:56,426 DEBUG [StoreFileCloser-5dc0d1664f37315aa7ff0a71dcd67668-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/family/8643f4c434754e938d50f24bc215eb2a 2024-11-24T08:54:56,426 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T08:54:56,426 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T08:54:56,426 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T08:54:56,426 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-24T08:54:56,431 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/default/testBlockEvictionOnGracefulStop1732438493598/5dc0d1664f37315aa7ff0a71dcd67668/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T08:54:56,432 INFO [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:56,432 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5dc0d1664f37315aa7ff0a71dcd67668: Waiting for close lock at 1732438496425Running coprocessor pre-close hooks at 1732438496425Disabling compacts and flushes for region at 1732438496425Disabling writes for close at 1732438496425Writing region close event to WAL at 1732438496426 (+1 ms)Running coprocessor post-close hooks at 1732438496432 (+6 ms)Closed at 1732438496432 2024-11-24T08:54:56,432 DEBUG [RS_CLOSE_REGION-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668. 2024-11-24T08:54:56,437 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 190dd986358f4da188e2e2217333bd24 2024-11-24T08:54:56,437 INFO [regionserver/469387a2cdb6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:54:56,449 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/info/190dd986358f4da188e2e2217333bd24 is 209, key is testBlockEvictionOnGracefulStop1732438493598,,1732438493602.5dc0d1664f37315aa7ff0a71dcd67668./info:regioninfo/1732438495860/Put/seqid=0 2024-11-24T08:54:56,450 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/info/190dd986358f4da188e2e2217333bd24 2024-11-24T08:54:56,457 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8ec67b445d224ef28c3291ca2fabc05b 2024-11-24T08:54:56,469 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/ns/8ec67b445d224ef28c3291ca2fabc05b is 43, key is default/ns:d/1732438493529/Put/seqid=0 2024-11-24T08:54:56,470 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/ns/8ec67b445d224ef28c3291ca2fabc05b 2024-11-24T08:54:56,474 INFO [regionserver/469387a2cdb6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:54:56,477 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 59121034535e4724a2d22cfd86a23ac6 2024-11-24T08:54:56,490 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/table/59121034535e4724a2d22cfd86a23ac6 is 80, key is testBlockEvictionOnGracefulStop1732438493598/table:state/1732438493980/Put/seqid=0 2024-11-24T08:54:56,490 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/table/59121034535e4724a2d22cfd86a23ac6 2024-11-24T08:54:56,492 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/info/190dd986358f4da188e2e2217333bd24 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/info/190dd986358f4da188e2e2217333bd24 2024-11-24T08:54:56,494 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/info/190dd986358f4da188e2e2217333bd24, delay=987 ms 2024-11-24T08:54:56,495 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/info/190dd986358f4da188e2e2217333bd24, entries=14, sequenceid=13, filesize=7.5 K 2024-11-24T08:54:56,495 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/ns/8ec67b445d224ef28c3291ca2fabc05b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/ns/8ec67b445d224ef28c3291ca2fabc05b 2024-11-24T08:54:56,496 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/ns/8ec67b445d224ef28c3291ca2fabc05b, delay=985 ms 2024-11-24T08:54:56,497 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/ns/8ec67b445d224ef28c3291ca2fabc05b, entries=2, sequenceid=13, filesize=5.0 K 2024-11-24T08:54:56,497 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/.tmp/table/59121034535e4724a2d22cfd86a23ac6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/table/59121034535e4724a2d22cfd86a23ac6 2024-11-24T08:54:56,499 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/table/59121034535e4724a2d22cfd86a23ac6, delay=933 ms 2024-11-24T08:54:56,500 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/table/59121034535e4724a2d22cfd86a23ac6, entries=2, sequenceid=13, filesize=5.3 K 2024-11-24T08:54:56,500 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false 2024-11-24T08:54:56,501 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/info/190dd986358f4da188e2e2217333bd24 2024-11-24T08:54:56,502 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/ns/8ec67b445d224ef28c3291ca2fabc05b 2024-11-24T08:54:56,502 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/table/59121034535e4724a2d22cfd86a23ac6 2024-11-24T08:54:56,506 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-24T08:54:56,506 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T08:54:56,506 INFO [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T08:54:56,506 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732438496426Running coprocessor pre-close hooks at 1732438496426Disabling compacts and flushes for region at 1732438496426Disabling writes for close at 1732438496426Obtaining lock to block concurrent updates at 1732438496426Preparing flush snapshotting stores in 1588230740 at 1732438496426Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732438496427 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732438496427Flushing 1588230740/info: creating writer at 1732438496427Flushing 1588230740/info: appending metadata at 1732438496449 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732438496449Flushing 1588230740/ns: creating writer at 1732438496452 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732438496469 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732438496469Flushing 1588230740/table: creating writer at 1732438496472 (+3 ms)Flushing 1588230740/table: appending metadata at 1732438496489 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732438496489Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16317523: reopening flushed file at 1732438496492 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@428f6eb0: reopening flushed file at 1732438496495 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b86767: reopening flushed file at 1732438496497 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false at 1732438496500 (+3 ms)Writing region close event to WAL at 1732438496502 (+2 ms)Running coprocessor post-close hooks at 1732438496506 (+4 ms)Closed at 1732438496506 2024-11-24T08:54:56,506 DEBUG [RS_CLOSE_META-regionserver/469387a2cdb6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T08:54:56,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40459-0x10070f0e2cc0003, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:56,522 INFO [RS:2;469387a2cdb6:40459 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:54:56,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40459-0x10070f0e2cc0003, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:56,523 INFO [RS:2;469387a2cdb6:40459 {}] regionserver.HRegionServer(1031): Exiting; stopping=469387a2cdb6,40459,1732438495804; zookeeper connection closed. 2024-11-24T08:54:56,523 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4497c668 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4497c668 2024-11-24T08:54:56,626 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(976): stopping server 469387a2cdb6,45065,1732438492830; all regions closed. 2024-11-24T08:54:56,631 DEBUG [RS:0;469387a2cdb6:45065 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs 2024-11-24T08:54:56,631 INFO [RS:0;469387a2cdb6:45065 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 469387a2cdb6%2C45065%2C1732438492830.meta:.meta(num 1732438493483) 2024-11-24T08:54:56,634 DEBUG [RS:0;469387a2cdb6:45065 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/oldWALs 2024-11-24T08:54:56,634 INFO [RS:0;469387a2cdb6:45065 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 469387a2cdb6%2C45065%2C1732438492830:(num 1732438493280) 2024-11-24T08:54:56,634 DEBUG [RS:0;469387a2cdb6:45065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:54:56,634 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:54:56,634 INFO [RS:0;469387a2cdb6:45065 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:54:56,635 INFO [RS:0;469387a2cdb6:45065 {}] hbase.ChoreService(370): Chore service for: regionserver/469387a2cdb6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T08:54:56,635 INFO [RS:0;469387a2cdb6:45065 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:54:56,635 INFO [regionserver/469387a2cdb6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:54:56,636 INFO [RS:0;469387a2cdb6:45065 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45065 2024-11-24T08:54:56,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:54:56,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/469387a2cdb6,45065,1732438492830 2024-11-24T08:54:56,638 INFO [RS:0;469387a2cdb6:45065 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:54:56,639 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [469387a2cdb6,45065,1732438492830] 2024-11-24T08:54:56,640 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/469387a2cdb6,45065,1732438492830 already deleted, retry=false 2024-11-24T08:54:56,640 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 469387a2cdb6,45065,1732438492830 expired; onlineServers=0 2024-11-24T08:54:56,640 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '469387a2cdb6,44433,1732438492780' ***** 2024-11-24T08:54:56,640 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T08:54:56,640 INFO [M:0;469387a2cdb6:44433 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:54:56,640 INFO [M:0;469387a2cdb6:44433 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:54:56,640 DEBUG [M:0;469387a2cdb6:44433 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T08:54:56,640 DEBUG [M:0;469387a2cdb6:44433 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T08:54:56,640 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T08:54:56,640 DEBUG [master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.large.0-1732438493069 {}] cleaner.HFileCleaner(306): Exit Thread[master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.large.0-1732438493069,5,FailOnTimeoutGroup] 2024-11-24T08:54:56,641 INFO [M:0;469387a2cdb6:44433 {}] hbase.ChoreService(370): Chore service for: master/469387a2cdb6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T08:54:56,641 INFO [M:0;469387a2cdb6:44433 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:54:56,641 DEBUG [master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.small.0-1732438493070 {}] cleaner.HFileCleaner(306): Exit Thread[master/469387a2cdb6:0:becomeActiveMaster-HFileCleaner.small.0-1732438493070,5,FailOnTimeoutGroup] 2024-11-24T08:54:56,641 DEBUG [M:0;469387a2cdb6:44433 {}] master.HMaster(1795): Stopping service threads 2024-11-24T08:54:56,641 INFO [M:0;469387a2cdb6:44433 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T08:54:56,641 INFO [M:0;469387a2cdb6:44433 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T08:54:56,641 INFO [M:0;469387a2cdb6:44433 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T08:54:56,641 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T08:54:56,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T08:54:56,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:54:56,642 DEBUG [M:0;469387a2cdb6:44433 {}] zookeeper.ZKUtil(347): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T08:54:56,642 WARN [M:0;469387a2cdb6:44433 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T08:54:56,642 INFO [M:0;469387a2cdb6:44433 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/.lastflushedseqids 2024-11-24T08:54:56,647 INFO [M:0;469387a2cdb6:44433 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T08:54:56,648 INFO [M:0;469387a2cdb6:44433 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T08:54:56,648 DEBUG [M:0;469387a2cdb6:44433 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T08:54:56,648 INFO [M:0;469387a2cdb6:44433 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:56,648 DEBUG [M:0;469387a2cdb6:44433 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:56,648 DEBUG [M:0;469387a2cdb6:44433 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T08:54:56,648 DEBUG [M:0;469387a2cdb6:44433 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:56,648 INFO [M:0;469387a2cdb6:44433 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-24T08:54:56,657 DEBUG [M:0;469387a2cdb6:44433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3e32a7ba7004451ca42c2a69059fedb7 2024-11-24T08:54:56,671 DEBUG [M:0;469387a2cdb6:44433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e32a7ba7004451ca42c2a69059fedb7 is 82, key is hbase:meta,,1/info:regioninfo/1732438493516/Put/seqid=0 2024-11-24T08:54:56,672 INFO [M:0;469387a2cdb6:44433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e32a7ba7004451ca42c2a69059fedb7 2024-11-24T08:54:56,679 DEBUG [M:0;469387a2cdb6:44433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 923e363d8c86409c8a8a75b03269b896 2024-11-24T08:54:56,692 DEBUG [M:0;469387a2cdb6:44433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/923e363d8c86409c8a8a75b03269b896 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732438493985/Put/seqid=0 2024-11-24T08:54:56,692 INFO [M:0;469387a2cdb6:44433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/923e363d8c86409c8a8a75b03269b896 2024-11-24T08:54:56,699 DEBUG [M:0;469387a2cdb6:44433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cf01b6222d4a48c395e360516b76eac1 2024-11-24T08:54:56,719 DEBUG [M:0;469387a2cdb6:44433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf01b6222d4a48c395e360516b76eac1 is 69, key is 469387a2cdb6,40459,1732438495804/rs:state/1732438495853/Put/seqid=0 2024-11-24T08:54:56,720 INFO [M:0;469387a2cdb6:44433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf01b6222d4a48c395e360516b76eac1 2024-11-24T08:54:56,722 INFO [M:0;469387a2cdb6:44433 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cf01b6222d4a48c395e360516b76eac1 2024-11-24T08:54:56,723 DEBUG [M:0;469387a2cdb6:44433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e32a7ba7004451ca42c2a69059fedb7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3e32a7ba7004451ca42c2a69059fedb7 2024-11-24T08:54:56,725 INFO [M:0;469387a2cdb6:44433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3e32a7ba7004451ca42c2a69059fedb7, entries=8, sequenceid=83, filesize=5.5 K 2024-11-24T08:54:56,726 DEBUG [M:0;469387a2cdb6:44433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/923e363d8c86409c8a8a75b03269b896 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/923e363d8c86409c8a8a75b03269b896 2024-11-24T08:54:56,728 INFO [M:0;469387a2cdb6:44433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/923e363d8c86409c8a8a75b03269b896, entries=9, sequenceid=83, filesize=6.4 K 2024-11-24T08:54:56,728 DEBUG [M:0;469387a2cdb6:44433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf01b6222d4a48c395e360516b76eac1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cf01b6222d4a48c395e360516b76eac1 2024-11-24T08:54:56,731 INFO [M:0;469387a2cdb6:44433 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cf01b6222d4a48c395e360516b76eac1 2024-11-24T08:54:56,731 INFO [M:0;469387a2cdb6:44433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb5e7b22-37ae-532d-e307-22dfa26cdba7/b50542c7-48cb-bf71-ad9d-123c77fd04ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cf01b6222d4a48c395e360516b76eac1, entries=3, sequenceid=83, filesize=5.3 K 2024-11-24T08:54:56,732 INFO [M:0;469387a2cdb6:44433 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=83, compaction requested=false 2024-11-24T08:54:56,733 INFO [M:0;469387a2cdb6:44433 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:54:56,734 DEBUG [M:0;469387a2cdb6:44433 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732438496648Disabling compacts and flushes for region at 1732438496648Disabling writes for close at 1732438496648Obtaining lock to block concurrent updates at 1732438496648Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732438496648Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1732438496649 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732438496649Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732438496649Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732438496671 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732438496671Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732438496673 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732438496691 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732438496691Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732438496694 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732438496718 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732438496718Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30ac0ac7: reopening flushed file at 1732438496723 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c669d2a: reopening flushed file at 1732438496726 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c5a9260: reopening flushed file at 1732438496728 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=83, compaction requested=false at 1732438496732 (+4 ms)Writing region close event to WAL at 1732438496733 (+1 ms)Closed at 1732438496733 2024-11-24T08:54:56,735 INFO [M:0;469387a2cdb6:44433 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T08:54:56,735 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:54:56,735 INFO [M:0;469387a2cdb6:44433 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44433 2024-11-24T08:54:56,735 INFO [M:0;469387a2cdb6:44433 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:54:56,739 INFO [RS:0;469387a2cdb6:45065 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:54:56,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:56,739 INFO [RS:0;469387a2cdb6:45065 {}] regionserver.HRegionServer(1031): Exiting; stopping=469387a2cdb6,45065,1732438492830; zookeeper connection closed. 2024-11-24T08:54:56,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45065-0x10070f0e2cc0001, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:56,740 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a814f34 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a814f34 2024-11-24T08:54:56,740 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T08:54:56,837 INFO [M:0;469387a2cdb6:44433 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:54:56,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:56,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44433-0x10070f0e2cc0000, quorum=127.0.0.1:62639, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:54:56,843 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T08:54:56,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T08:54:56,845 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:54:56,853 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=2147 (was 2222)