2024-12-09 14:59:11,077 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-09 14:59:11,093 main DEBUG Took 0.014617 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 14:59:11,094 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 14:59:11,094 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 14:59:11,095 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 14:59:11,096 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,103 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 14:59:11,117 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,118 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,118 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,118 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,119 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,119 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,120 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,120 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,121 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,121 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,122 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,123 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,123 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,123 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,124 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,124 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,125 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,125 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,125 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,126 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,126 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,127 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:59:11,127 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,127 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 14:59:11,129 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:59:11,131 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 14:59:11,133 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 14:59:11,134 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 14:59:11,136 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 14:59:11,136 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 14:59:11,146 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 14:59:11,150 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 14:59:11,152 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 14:59:11,152 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 14:59:11,153 main DEBUG createAppenders(={Console}) 2024-12-09 14:59:11,154 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-09 14:59:11,154 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-09 14:59:11,155 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-09 14:59:11,155 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 14:59:11,156 main DEBUG OutputStream closed 2024-12-09 14:59:11,156 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 14:59:11,156 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 14:59:11,157 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-09 14:59:11,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 14:59:11,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 14:59:11,221 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 14:59:11,222 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 14:59:11,222 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 14:59:11,222 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 14:59:11,222 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 14:59:11,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 14:59:11,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 14:59:11,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 14:59:11,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 14:59:11,224 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 14:59:11,224 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 14:59:11,224 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 14:59:11,225 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 14:59:11,225 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 14:59:11,225 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 14:59:11,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 14:59:11,227 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 14:59:11,228 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-09 14:59:11,228 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 14:59:11,228 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-09T14:59:11,239 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-09 14:59:11,241 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 14:59:11,241 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T14:59:11,466 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b 2024-12-09T14:59:11,485 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=309, ProcessCount=11, AvailableMemoryMB=8125 2024-12-09T14:59:11,575 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T14:59:11,653 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:11,667 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/cluster_0d4878b9-58db-31a3-0df6-9ca098a3082f, deleteOnExit=true 2024-12-09T14:59:11,749 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/cluster_0d4878b9-58db-31a3-0df6-9ca098a3082f/zookeeper_0, clientPort=50273, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/cluster_0d4878b9-58db-31a3-0df6-9ca098a3082f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/cluster_0d4878b9-58db-31a3-0df6-9ca098a3082f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T14:59:11,760 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50273 2024-12-09T14:59:11,760 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:11,835 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a with version=8 2024-12-09T14:59:11,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/hbase-staging 2024-12-09T14:59:11,911 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T14:59:12,164 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2491923345a5:0 server-side Connection retries=45 2024-12-09T14:59:12,172 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:12,173 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:12,180 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:59:12,180 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:12,180 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:59:12,291 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T14:59:12,365 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T14:59:12,399 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T14:59:12,408 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T14:59:12,412 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:59:12,446 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 111286 (auto-detected) 2024-12-09T14:59:12,447 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T14:59:12,468 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34921 2024-12-09T14:59:12,495 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34921 connecting to ZooKeeper ensemble=127.0.0.1:50273 2024-12-09T14:59:12,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:349210x0, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:59:12,603 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34921-0x1000b24338f0000 connected 2024-12-09T14:59:12,710 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:12,711 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:12,719 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:12,723 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a, hbase.cluster.distributed=false 2024-12-09T14:59:12,749 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:59:12,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34921 2024-12-09T14:59:12,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34921 2024-12-09T14:59:12,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34921 2024-12-09T14:59:12,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34921 2024-12-09T14:59:12,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34921 2024-12-09T14:59:12,833 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2491923345a5:0 server-side Connection retries=45 2024-12-09T14:59:12,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:12,835 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:12,835 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:59:12,835 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:12,835 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:59:12,838 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T14:59:12,840 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:59:12,841 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38515 2024-12-09T14:59:12,843 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38515 connecting to ZooKeeper ensemble=127.0.0.1:50273 2024-12-09T14:59:12,843 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:12,844 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:12,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385150x0, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:59:12,855 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:385150x0, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:12,855 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38515-0x1000b24338f0001 connected 2024-12-09T14:59:12,860 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T14:59:12,875 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/bucket.cache 2024-12-09T14:59:12,882 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-09T14:59:12,884 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-09T14:59:12,886 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-09T14:59:12,888 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-09T14:59:12,888 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-09T14:59:12,894 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-09T14:59:12,896 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-09T14:59:12,897 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/bucket.persistence is complete. 2024-12-09T14:59:12,902 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T14:59:12,905 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T14:59:12,911 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:59:12,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38515 2024-12-09T14:59:12,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38515 2024-12-09T14:59:12,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38515 2024-12-09T14:59:12,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38515 2024-12-09T14:59:12,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38515 2024-12-09T14:59:12,942 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2491923345a5:0 server-side Connection retries=45 2024-12-09T14:59:12,942 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:12,942 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:12,943 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:59:12,943 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:12,943 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:59:12,944 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T14:59:12,944 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:59:12,945 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43285 2024-12-09T14:59:12,948 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43285 connecting to ZooKeeper ensemble=127.0.0.1:50273 2024-12-09T14:59:12,949 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:12,950 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:12,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432850x0, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:59:12,989 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:12,989 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43285-0x1000b24338f0002 connected 2024-12-09T14:59:12,990 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T14:59:12,991 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/bucket.cache 2024-12-09T14:59:12,991 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-09T14:59:12,991 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-09T14:59:12,992 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-09T14:59:12,992 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-09T14:59:12,992 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-09T14:59:12,993 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-09T14:59:12,993 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-09T14:59:12,993 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/bucket.persistence is complete. 2024-12-09T14:59:12,994 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T14:59:12,995 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T14:59:12,998 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:59:12,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43285 2024-12-09T14:59:12,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43285 2024-12-09T14:59:13,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43285 2024-12-09T14:59:13,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43285 2024-12-09T14:59:13,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43285 2024-12-09T14:59:13,024 DEBUG [M:0;2491923345a5:34921 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2491923345a5:34921 2024-12-09T14:59:13,025 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2491923345a5,34921,1733756352010 2024-12-09T14:59:13,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:13,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:13,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:13,040 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2491923345a5,34921,1733756352010 2024-12-09T14:59:13,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T14:59:13,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T14:59:13,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,070 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T14:59:13,071 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2491923345a5,34921,1733756352010 from backup master directory 2024-12-09T14:59:13,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:13,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2491923345a5,34921,1733756352010 2024-12-09T14:59:13,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:13,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:13,080 WARN [master/2491923345a5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:59:13,080 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2491923345a5,34921,1733756352010 2024-12-09T14:59:13,082 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T14:59:13,084 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T14:59:13,091 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/hbase.id] with ID: b0e674c5-788f-4076-baf5-8c61137e636e 2024-12-09T14:59:13,091 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/.tmp/hbase.id 2024-12-09T14:59:13,096 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/hbase.id] 2024-12-09T14:59:13,130 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:13,137 INFO [master/2491923345a5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T14:59:13,161 INFO [master/2491923345a5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-12-09T14:59:13,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,201 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T14:59:13,204 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T14:59:13,240 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:59:13,246 INFO [master/2491923345a5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:59:13,292 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store 2024-12-09T14:59:13,312 INFO [master/2491923345a5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T14:59:13,315 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:13,317 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T14:59:13,317 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:13,317 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:13,319 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T14:59:13,319 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:13,320 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:13,320 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733756353317Disabling compacts and flushes for region at 1733756353317Disabling writes for close at 1733756353319 (+2 ms)Writing region close event to WAL at 1733756353319Closed at 1733756353320 (+1 ms) 2024-12-09T14:59:13,321 WARN [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/.initializing 2024-12-09T14:59:13,321 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/WALs/2491923345a5,34921,1733756352010 2024-12-09T14:59:13,334 INFO [master/2491923345a5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:59:13,348 INFO [master/2491923345a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2491923345a5%2C34921%2C1733756352010, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/WALs/2491923345a5,34921,1733756352010, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/oldWALs, maxLogs=10 2024-12-09T14:59:13,383 INFO [master/2491923345a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/WALs/2491923345a5,34921,1733756352010/2491923345a5%2C34921%2C1733756352010.1733756353352 2024-12-09T14:59:13,384 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T14:59:13,384 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:59:13,385 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:13,388 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,389 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,423 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,460 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T14:59:13,466 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:13,469 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:13,469 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T14:59:13,474 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:13,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:59:13,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T14:59:13,480 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:13,481 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:59:13,481 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T14:59:13,485 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:13,486 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:59:13,486 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,490 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,490 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,493 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,494 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,496 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:59:13,498 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:13,506 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:59:13,507 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66452589, jitterRate=-0.00977925956249237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:59:13,516 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733756353401Initializing all the Stores at 1733756353401Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756353402 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756353402Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756353403 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756353403Cleaning up temporary data from old regions at 1733756353494 (+91 ms)Region opened successfully at 1733756353516 (+22 ms) 2024-12-09T14:59:13,518 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T14:59:13,557 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36085e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2491923345a5/172.17.0.2:0 2024-12-09T14:59:13,592 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T14:59:13,607 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T14:59:13,608 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T14:59:13,612 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T14:59:13,613 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T14:59:13,621 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-09T14:59:13,621 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T14:59:13,647 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T14:59:13,658 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T14:59:13,704 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T14:59:13,708 INFO [master/2491923345a5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T14:59:13,710 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T14:59:13,720 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T14:59:13,724 INFO [master/2491923345a5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T14:59:13,729 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T14:59:13,737 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T14:59:13,740 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T14:59:13,751 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T14:59:13,773 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T14:59:13,784 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T14:59:13,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:13,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:13,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:13,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,813 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2491923345a5,34921,1733756352010, sessionid=0x1000b24338f0000, setting cluster-up flag (Was=false) 2024-12-09T14:59:13,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,862 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T14:59:13,867 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2491923345a5,34921,1733756352010 2024-12-09T14:59:13,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:13,920 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T14:59:13,923 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2491923345a5,34921,1733756352010 2024-12-09T14:59:13,930 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T14:59:13,997 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T14:59:14,006 INFO [master/2491923345a5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T14:59:14,013 INFO [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(746): ClusterId : b0e674c5-788f-4076-baf5-8c61137e636e 2024-12-09T14:59:14,013 INFO [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(746): ClusterId : b0e674c5-788f-4076-baf5-8c61137e636e 2024-12-09T14:59:14,013 INFO [master/2491923345a5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T14:59:14,015 DEBUG [RS:1;2491923345a5:43285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T14:59:14,015 DEBUG [RS:0;2491923345a5:38515 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T14:59:14,019 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2491923345a5,34921,1733756352010 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T14:59:14,040 DEBUG [RS:0;2491923345a5:38515 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T14:59:14,040 DEBUG [RS:1;2491923345a5:43285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T14:59:14,040 DEBUG [RS:0;2491923345a5:38515 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T14:59:14,040 DEBUG [RS:1;2491923345a5:43285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T14:59:14,041 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:59:14,041 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:59:14,041 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:59:14,041 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:59:14,042 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2491923345a5:0, corePoolSize=10, maxPoolSize=10 2024-12-09T14:59:14,042 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,042 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2491923345a5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:59:14,043 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,044 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733756384044 2024-12-09T14:59:14,046 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T14:59:14,046 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:59:14,047 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T14:59:14,047 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T14:59:14,050 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T14:59:14,050 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T14:59:14,050 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T14:59:14,050 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T14:59:14,051 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:14,051 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T14:59:14,051 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,054 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T14:59:14,056 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T14:59:14,056 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T14:59:14,058 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T14:59:14,058 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T14:59:14,060 DEBUG [RS:0;2491923345a5:38515 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T14:59:14,060 DEBUG [RS:1;2491923345a5:43285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T14:59:14,060 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.large.0-1733756354059,5,FailOnTimeoutGroup] 2024-12-09T14:59:14,061 DEBUG [RS:1;2491923345a5:43285 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c7468b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2491923345a5/172.17.0.2:0 2024-12-09T14:59:14,061 DEBUG [RS:0;2491923345a5:38515 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e437646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2491923345a5/172.17.0.2:0 2024-12-09T14:59:14,063 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.small.0-1733756354061,5,FailOnTimeoutGroup] 2024-12-09T14:59:14,063 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,063 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T14:59:14,064 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,065 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,071 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T14:59:14,072 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a 2024-12-09T14:59:14,080 DEBUG [RS:1;2491923345a5:43285 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2491923345a5:43285 2024-12-09T14:59:14,081 DEBUG [RS:0;2491923345a5:38515 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2491923345a5:38515 2024-12-09T14:59:14,083 INFO [RS:1;2491923345a5:43285 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T14:59:14,083 INFO [RS:0;2491923345a5:38515 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T14:59:14,083 INFO [RS:1;2491923345a5:43285 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T14:59:14,083 INFO [RS:0;2491923345a5:38515 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T14:59:14,083 DEBUG [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T14:59:14,083 DEBUG [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T14:59:14,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:14,086 INFO [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(2659): reportForDuty to master=2491923345a5,34921,1733756352010 with port=38515, startcode=1733756352800 2024-12-09T14:59:14,086 INFO [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(2659): reportForDuty to master=2491923345a5,34921,1733756352010 with port=43285, startcode=1733756352941 2024-12-09T14:59:14,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-09T14:59:14,091 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T14:59:14,092 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:14,092 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:14,092 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-09T14:59:14,096 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T14:59:14,096 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:14,096 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:14,097 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-09T14:59:14,099 DEBUG [RS:0;2491923345a5:38515 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T14:59:14,099 DEBUG [RS:1;2491923345a5:43285 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T14:59:14,103 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T14:59:14,103 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:14,104 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:14,104 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-09T14:59:14,111 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T14:59:14,111 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:14,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:14,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T14:59:14,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740 2024-12-09T14:59:14,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740 2024-12-09T14:59:14,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T14:59:14,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T14:59:14,113 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:59:14,114 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T14:59:14,121 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:59:14,121 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73675309, jitterRate=0.09784765541553497}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:59:14,124 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733756354086Initializing all the Stores at 1733756354087 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756354087Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756354087Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756354087Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756354087Cleaning up temporary data from old regions at 1733756354113 (+26 ms)Region opened successfully at 1733756354124 (+11 ms) 2024-12-09T14:59:14,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T14:59:14,125 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T14:59:14,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T14:59:14,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T14:59:14,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T14:59:14,127 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T14:59:14,127 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733756354125Disabling compacts and flushes for region at 1733756354125Disabling writes for close at 1733756354125Writing region close event to WAL at 1733756354126 (+1 ms)Closed at 1733756354127 (+1 ms) 2024-12-09T14:59:14,129 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:59:14,129 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T14:59:14,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T14:59:14,149 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T14:59:14,152 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T14:59:14,168 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41449, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T14:59:14,168 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39589, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T14:59:14,174 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34921 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2491923345a5,38515,1733756352800 2024-12-09T14:59:14,176 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34921 {}] master.ServerManager(517): Registering regionserver=2491923345a5,38515,1733756352800 2024-12-09T14:59:14,187 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34921 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2491923345a5,43285,1733756352941 2024-12-09T14:59:14,187 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34921 {}] master.ServerManager(517): Registering regionserver=2491923345a5,43285,1733756352941 2024-12-09T14:59:14,191 DEBUG [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a 2024-12-09T14:59:14,191 DEBUG [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a 2024-12-09T14:59:14,191 DEBUG [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-09T14:59:14,191 DEBUG [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-09T14:59:14,191 DEBUG [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T14:59:14,191 DEBUG [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T14:59:14,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:59:14,240 DEBUG [RS:1;2491923345a5:43285 {}] zookeeper.ZKUtil(111): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2491923345a5,43285,1733756352941 2024-12-09T14:59:14,241 WARN [RS:1;2491923345a5:43285 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:59:14,241 DEBUG [RS:0;2491923345a5:38515 {}] zookeeper.ZKUtil(111): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2491923345a5,38515,1733756352800 2024-12-09T14:59:14,241 INFO [RS:1;2491923345a5:43285 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:59:14,241 WARN [RS:0;2491923345a5:38515 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:59:14,241 INFO [RS:0;2491923345a5:38515 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:59:14,241 DEBUG [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,43285,1733756352941 2024-12-09T14:59:14,241 DEBUG [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,38515,1733756352800 2024-12-09T14:59:14,244 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2491923345a5,43285,1733756352941] 2024-12-09T14:59:14,244 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2491923345a5,38515,1733756352800] 2024-12-09T14:59:14,273 INFO [RS:1;2491923345a5:43285 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T14:59:14,273 INFO [RS:0;2491923345a5:38515 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T14:59:14,293 INFO [RS:0;2491923345a5:38515 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T14:59:14,293 INFO [RS:1;2491923345a5:43285 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T14:59:14,297 INFO [RS:0;2491923345a5:38515 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:59:14,298 INFO [RS:1;2491923345a5:43285 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:59:14,298 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,298 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,299 INFO [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T14:59:14,299 INFO [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T14:59:14,303 WARN [2491923345a5:34921 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T14:59:14,303 INFO [RS:0;2491923345a5:38515 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T14:59:14,303 INFO [RS:1;2491923345a5:43285 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T14:59:14,305 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,305 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,305 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,305 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,305 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,305 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,305 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,305 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,305 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,306 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,306 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,306 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:59:14,306 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,306 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,306 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:59:14,306 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,306 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,306 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,306 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,306 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,306 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,307 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,307 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,307 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:59:14,307 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,307 DEBUG [RS:0;2491923345a5:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:59:14,307 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,307 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:14,307 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:59:14,307 DEBUG [RS:1;2491923345a5:43285 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:59:14,311 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,311 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,311 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,311 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,311 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,311 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,311 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,311 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,311 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,311 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,312 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,38515,1733756352800-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:59:14,312 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,43285,1733756352941-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:59:14,332 INFO [RS:1;2491923345a5:43285 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T14:59:14,332 INFO [RS:0;2491923345a5:38515 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T14:59:14,334 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,43285,1733756352941-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,334 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,38515,1733756352800-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,334 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,334 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,334 INFO [RS:1;2491923345a5:43285 {}] regionserver.Replication(171): 2491923345a5,43285,1733756352941 started 2024-12-09T14:59:14,334 INFO [RS:0;2491923345a5:38515 {}] regionserver.Replication(171): 2491923345a5,38515,1733756352800 started 2024-12-09T14:59:14,352 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,353 INFO [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(1482): Serving as 2491923345a5,43285,1733756352941, RpcServer on 2491923345a5/172.17.0.2:43285, sessionid=0x1000b24338f0002 2024-12-09T14:59:14,354 DEBUG [RS:1;2491923345a5:43285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T14:59:14,354 DEBUG [RS:1;2491923345a5:43285 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2491923345a5,43285,1733756352941 2024-12-09T14:59:14,354 DEBUG [RS:1;2491923345a5:43285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,43285,1733756352941' 2024-12-09T14:59:14,354 DEBUG [RS:1;2491923345a5:43285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T14:59:14,355 DEBUG [RS:1;2491923345a5:43285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T14:59:14,356 DEBUG [RS:1;2491923345a5:43285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T14:59:14,356 DEBUG [RS:1;2491923345a5:43285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T14:59:14,357 DEBUG [RS:1;2491923345a5:43285 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2491923345a5,43285,1733756352941 2024-12-09T14:59:14,357 DEBUG [RS:1;2491923345a5:43285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,43285,1733756352941' 2024-12-09T14:59:14,357 DEBUG [RS:1;2491923345a5:43285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T14:59:14,358 DEBUG [RS:1;2491923345a5:43285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T14:59:14,358 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:14,358 INFO [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(1482): Serving as 2491923345a5,38515,1733756352800, RpcServer on 2491923345a5/172.17.0.2:38515, sessionid=0x1000b24338f0001 2024-12-09T14:59:14,358 DEBUG [RS:0;2491923345a5:38515 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T14:59:14,358 DEBUG [RS:0;2491923345a5:38515 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2491923345a5,38515,1733756352800 2024-12-09T14:59:14,358 DEBUG [RS:0;2491923345a5:38515 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,38515,1733756352800' 2024-12-09T14:59:14,358 DEBUG [RS:1;2491923345a5:43285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T14:59:14,358 DEBUG [RS:0;2491923345a5:38515 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T14:59:14,358 INFO [RS:1;2491923345a5:43285 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T14:59:14,359 INFO [RS:1;2491923345a5:43285 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T14:59:14,359 DEBUG [RS:0;2491923345a5:38515 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T14:59:14,360 DEBUG [RS:0;2491923345a5:38515 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T14:59:14,360 DEBUG [RS:0;2491923345a5:38515 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T14:59:14,360 DEBUG [RS:0;2491923345a5:38515 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2491923345a5,38515,1733756352800 2024-12-09T14:59:14,360 DEBUG [RS:0;2491923345a5:38515 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,38515,1733756352800' 2024-12-09T14:59:14,360 DEBUG [RS:0;2491923345a5:38515 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T14:59:14,360 DEBUG [RS:0;2491923345a5:38515 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T14:59:14,361 DEBUG [RS:0;2491923345a5:38515 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T14:59:14,361 INFO [RS:0;2491923345a5:38515 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T14:59:14,361 INFO [RS:0;2491923345a5:38515 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T14:59:14,463 INFO [RS:1;2491923345a5:43285 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:59:14,463 INFO [RS:0;2491923345a5:38515 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:59:14,464 INFO [RS:1;2491923345a5:43285 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2491923345a5%2C43285%2C1733756352941, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,43285,1733756352941, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs, maxLogs=55 2024-12-09T14:59:14,464 INFO [RS:0;2491923345a5:38515 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2491923345a5%2C38515%2C1733756352800, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,38515,1733756352800, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs, maxLogs=55 2024-12-09T14:59:14,483 INFO [RS:0;2491923345a5:38515 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,38515,1733756352800/2491923345a5%2C38515%2C1733756352800.1733756354467 2024-12-09T14:59:14,483 DEBUG [RS:0;2491923345a5:38515 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T14:59:14,489 INFO [RS:1;2491923345a5:43285 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,43285,1733756352941/2491923345a5%2C43285%2C1733756352941.1733756354467 2024-12-09T14:59:14,489 DEBUG [RS:1;2491923345a5:43285 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T14:59:14,557 DEBUG [2491923345a5:34921 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-09T14:59:14,572 DEBUG [2491923345a5:34921 {}] balancer.BalancerClusterState(204): Hosts are {2491923345a5=0} racks are {/default-rack=0} 2024-12-09T14:59:14,575 DEBUG [2491923345a5:34921 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T14:59:14,575 DEBUG [2491923345a5:34921 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T14:59:14,575 DEBUG [2491923345a5:34921 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T14:59:14,575 DEBUG [2491923345a5:34921 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T14:59:14,575 INFO [2491923345a5:34921 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T14:59:14,576 INFO [2491923345a5:34921 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T14:59:14,576 DEBUG [2491923345a5:34921 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T14:59:14,582 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2491923345a5,43285,1733756352941 2024-12-09T14:59:14,587 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2491923345a5,43285,1733756352941, state=OPENING 2024-12-09T14:59:14,654 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T14:59:14,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:14,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:14,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:14,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:14,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:14,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:14,669 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T14:59:14,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2491923345a5,43285,1733756352941}] 2024-12-09T14:59:14,846 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:59:14,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44477, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:59:14,857 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T14:59:14,857 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:59:14,858 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T14:59:14,859 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2491923345a5%2C43285%2C1733756352941.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,43285,1733756352941, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs, maxLogs=55 2024-12-09T14:59:14,877 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,43285,1733756352941/2491923345a5%2C43285%2C1733756352941.meta.1733756354861.meta 2024-12-09T14:59:14,878 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T14:59:14,878 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:59:14,881 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T14:59:14,884 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T14:59:14,890 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T14:59:14,895 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T14:59:14,895 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:14,896 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T14:59:14,896 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T14:59:14,898 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-09T14:59:14,899 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T14:59:14,899 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:14,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:14,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-09T14:59:14,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T14:59:14,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:14,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:14,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-09T14:59:14,902 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T14:59:14,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:14,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:14,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-09T14:59:14,902 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T14:59:14,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:14,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:14,903 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T14:59:14,904 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740 2024-12-09T14:59:14,905 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740 2024-12-09T14:59:14,905 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T14:59:14,905 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T14:59:14,905 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:59:14,906 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T14:59:14,907 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71728384, jitterRate=0.06883621215820312}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:59:14,907 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T14:59:14,908 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733756354897Writing region info on filesystem at 1733756354897Initializing all the Stores at 1733756354897Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756354897Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756354898 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756354898Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756354898Cleaning up temporary data from old regions at 1733756354905 (+7 ms)Running coprocessor post-open hooks at 1733756354907 (+2 ms)Region opened successfully at 1733756354908 (+1 ms) 2024-12-09T14:59:14,914 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733756354839 2024-12-09T14:59:14,922 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T14:59:14,922 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T14:59:14,923 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2491923345a5,43285,1733756352941 2024-12-09T14:59:14,924 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2491923345a5,43285,1733756352941, state=OPEN 2024-12-09T14:59:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:59:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:59:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:59:14,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:14,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:14,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:14,947 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2491923345a5,43285,1733756352941 2024-12-09T14:59:14,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T14:59:14,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2491923345a5,43285,1733756352941 in 276 msec 2024-12-09T14:59:14,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T14:59:14,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 816 msec 2024-12-09T14:59:14,958 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:59:14,958 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T14:59:14,978 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T14:59:14,979 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2491923345a5,43285,1733756352941, seqNum=-1] 2024-12-09T14:59:15,002 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:59:15,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44067, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:59:15,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0650 sec 2024-12-09T14:59:15,028 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733756355028, completionTime=-1 2024-12-09T14:59:15,031 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-09T14:59:15,031 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T14:59:15,053 INFO [master/2491923345a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T14:59:15,053 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733756415053 2024-12-09T14:59:15,053 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733756475053 2024-12-09T14:59:15,053 INFO [master/2491923345a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-12-09T14:59:15,056 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T14:59:15,061 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,34921,1733756352010-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:15,062 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,34921,1733756352010-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:15,062 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,34921,1733756352010-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:15,063 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2491923345a5:34921, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:15,064 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:15,064 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:15,071 DEBUG [master/2491923345a5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T14:59:15,096 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.014sec 2024-12-09T14:59:15,097 INFO [master/2491923345a5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T14:59:15,098 INFO [master/2491923345a5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T14:59:15,099 INFO [master/2491923345a5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T14:59:15,099 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T14:59:15,100 INFO [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T14:59:15,100 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,34921,1733756352010-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:59:15,101 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,34921,1733756352010-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T14:59:15,102 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T14:59:15,103 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T14:59:15,103 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,34921,1733756352010-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:15,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:59:15,123 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T14:59:15,123 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T14:59:15,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2491923345a5,34921,-1 for getting cluster id 2024-12-09T14:59:15,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T14:59:15,140 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b0e674c5-788f-4076-baf5-8c61137e636e' 2024-12-09T14:59:15,142 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T14:59:15,142 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b0e674c5-788f-4076-baf5-8c61137e636e" 2024-12-09T14:59:15,143 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77a1369a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:59:15,143 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2491923345a5,34921,-1] 2024-12-09T14:59:15,145 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T14:59:15,146 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:15,148 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T14:59:15,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:59:15,151 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T14:59:15,157 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2491923345a5,43285,1733756352941, seqNum=-1] 2024-12-09T14:59:15,157 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:59:15,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:59:15,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2491923345a5,34921,1733756352010 2024-12-09T14:59:15,193 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T14:59:15,199 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2491923345a5,34921,1733756352010 2024-12-09T14:59:15,202 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3bf72b25 2024-12-09T14:59:15,203 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T14:59:15,205 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T14:59:15,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733756355190', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T14:59:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733756355190 2024-12-09T14:59:15,218 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733756355190 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T14:59:15,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733756355190" procId is: 4 2024-12-09T14:59:15,220 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:15,222 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733756355190 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T14:59:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:59:15,237 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733756355190-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9710971f63f118d0433adb842186bd89, NAME => 'testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733756355190', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a 2024-12-09T14:59:15,243 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733756355190-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:15,243 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733756355190-pool-0 {}] regionserver.HRegion(1722): Closing 9710971f63f118d0433adb842186bd89, disabling compactions & flushes 2024-12-09T14:59:15,243 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733756355190-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:15,243 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733756355190-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:15,243 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733756355190-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. after waiting 0 ms 2024-12-09T14:59:15,244 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733756355190-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:15,244 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733756355190-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:15,244 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733756355190-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9710971f63f118d0433adb842186bd89: Waiting for close lock at 1733756355243Disabling compacts and flushes for region at 1733756355243Disabling writes for close at 1733756355243Writing region close event to WAL at 1733756355244 (+1 ms)Closed at 1733756355244 2024-12-09T14:59:15,245 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733756355190 execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T14:59:15,251 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733756355246"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733756355246"}]},"ts":"1733756355246"} 2024-12-09T14:59:15,256 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T14:59:15,257 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733756355190 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T14:59:15,260 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733756355190","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733756355257"}]},"ts":"1733756355257"} 2024-12-09T14:59:15,264 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733756355190, state=ENABLING in hbase:meta 2024-12-09T14:59:15,264 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2491923345a5=0} racks are {/default-rack=0} 2024-12-09T14:59:15,265 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T14:59:15,265 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T14:59:15,265 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T14:59:15,265 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T14:59:15,265 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T14:59:15,265 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T14:59:15,265 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T14:59:15,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733756355190, region=9710971f63f118d0433adb842186bd89, ASSIGN}] 2024-12-09T14:59:15,268 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733756355190, region=9710971f63f118d0433adb842186bd89, ASSIGN 2024-12-09T14:59:15,269 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733756355190, region=9710971f63f118d0433adb842186bd89, ASSIGN; state=OFFLINE, location=2491923345a5,43285,1733756352941; forceNewPlan=false, retain=false 2024-12-09T14:59:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:59:15,426 INFO [2491923345a5:34921 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T14:59:15,428 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9710971f63f118d0433adb842186bd89, regionState=OPENING, regionLocation=2491923345a5,43285,1733756352941 2024-12-09T14:59:15,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733756355190, region=9710971f63f118d0433adb842186bd89, ASSIGN because future has completed 2024-12-09T14:59:15,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9710971f63f118d0433adb842186bd89, server=2491923345a5,43285,1733756352941}] 2024-12-09T14:59:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:59:15,593 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:15,594 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9710971f63f118d0433adb842186bd89, NAME => 'testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:59:15,594 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733756355190 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:15,594 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:15,594 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:15,594 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:15,595 INFO [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:15,598 INFO [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9710971f63f118d0433adb842186bd89 columnFamilyName family 2024-12-09T14:59:15,598 DEBUG [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:15,599 INFO [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] regionserver.HStore(327): Store=9710971f63f118d0433adb842186bd89/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:59:15,599 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:15,600 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89 2024-12-09T14:59:15,600 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89 2024-12-09T14:59:15,600 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:15,600 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:15,601 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:15,606 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:59:15,607 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9710971f63f118d0433adb842186bd89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69929105, jitterRate=0.04202486574649811}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:59:15,607 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:15,608 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9710971f63f118d0433adb842186bd89: Running coprocessor pre-open hook at 1733756355594Writing region info on filesystem at 1733756355594Initializing all the Stores at 1733756355595 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756355595Cleaning up temporary data from old regions at 1733756355600 (+5 ms)Running coprocessor post-open hooks at 1733756355607 (+7 ms)Region opened successfully at 1733756355608 (+1 ms) 2024-12-09T14:59:15,609 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89., pid=6, masterSystemTime=1733756355590 2024-12-09T14:59:15,613 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:15,613 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:15,613 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9710971f63f118d0433adb842186bd89, regionState=OPEN, openSeqNum=2, regionLocation=2491923345a5,43285,1733756352941 2024-12-09T14:59:15,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9710971f63f118d0433adb842186bd89, server=2491923345a5,43285,1733756352941 because future has completed 2024-12-09T14:59:15,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T14:59:15,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9710971f63f118d0433adb842186bd89, server=2491923345a5,43285,1733756352941 in 182 msec 2024-12-09T14:59:15,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T14:59:15,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733756355190, region=9710971f63f118d0433adb842186bd89, ASSIGN in 354 msec 2024-12-09T14:59:15,624 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733756355190 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T14:59:15,624 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733756355190","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733756355624"}]},"ts":"1733756355624"} 2024-12-09T14:59:15,627 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733756355190, state=ENABLED in hbase:meta 2024-12-09T14:59:15,628 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733756355190 execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T14:59:15,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733756355190 in 415 msec 2024-12-09T14:59:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:59:15,858 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733756355190 get assigned. Timeout = 60000ms 2024-12-09T14:59:15,858 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733756355190 completed 2024-12-09T14:59:15,862 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T14:59:15,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733756355190 assigned to meta. Checking AM states. 2024-12-09T14:59:15,870 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T14:59:15,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733756355190 assigned. 2024-12-09T14:59:15,880 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733756355190', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89., hostname=2491923345a5,43285,1733756352941, seqNum=2] 2024-12-09T14:59:15,894 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9710971f63f118d0433adb842186bd89 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-09T14:59:15,917 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 46610c0fecde4828bfc867b0e62799e6 2024-12-09T14:59:15,950 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/.tmp/family/46610c0fecde4828bfc867b0e62799e6 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-09T14:59:15,954 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/.tmp/family/46610c0fecde4828bfc867b0e62799e6 2024-12-09T14:59:15,983 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/.tmp/family/46610c0fecde4828bfc867b0e62799e6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6 2024-12-09T14:59:15,984 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6, delay=924 ms 2024-12-09T14:59:15,985 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6, entries=2, sequenceid=6, filesize=4.9 K 2024-12-09T14:59:15,991 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 9710971f63f118d0433adb842186bd89 in 98ms, sequenceid=6, compaction requested=false 2024-12-09T14:59:15,992 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733756355190' 2024-12-09T14:59:15,994 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9710971f63f118d0433adb842186bd89: 2024-12-09T14:59:16,911 DEBUG [hfile-prefetch-1733756355984 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46610c0fecde4828bfc867b0e62799e6_0. 2024-12-09T14:59:16,917 DEBUG [hfile-prefetch-1733756355984 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46610c0fecde4828bfc867b0e62799e6_117. 2024-12-09T14:59:16,918 DEBUG [hfile-prefetch-1733756355984 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6, with total blocks 2, and data blocks 1 2024-12-09T14:59:16,918 DEBUG [hfile-prefetch-1733756355984 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-09T14:59:16,918 DEBUG [hfile-prefetch-1733756355984 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-09T14:59:16,919 DEBUG [hfile-prefetch-1733756355984 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6. Will sleep 100ms and try the verification again. 2024-12-09T14:59:16,995 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-09T14:59:16,996 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-09T14:59:16,997 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T14:59:16,997 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T14:59:17,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=9710971f63f118d0433adb842186bd89, source=2491923345a5,43285,1733756352941, destination=2491923345a5,38515,1733756352800, warming up region on 2491923345a5,38515,1733756352800 2024-12-09T14:59:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:59:17,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=9710971f63f118d0433adb842186bd89, source=2491923345a5,43285,1733756352941, destination=2491923345a5,38515,1733756352800, running balancer 2024-12-09T14:59:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733756355190, region=9710971f63f118d0433adb842186bd89, REOPEN/MOVE 2024-12-09T14:59:17,013 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733756355190, region=9710971f63f118d0433adb842186bd89, REOPEN/MOVE 2024-12-09T14:59:17,015 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9710971f63f118d0433adb842186bd89, regionState=CLOSING, regionLocation=2491923345a5,43285,1733756352941 2024-12-09T14:59:17,016 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-09T14:59:17,016 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733756356996 2024-12-09T14:59:17,017 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-09T14:59:17,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44885, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:59:17,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733756355190, region=9710971f63f118d0433adb842186bd89, REOPEN/MOVE because future has completed 2024-12-09T14:59:17,018 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-09T14:59:17,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9710971f63f118d0433adb842186bd89, server=2491923345a5,43285,1733756352941}] 2024-12-09T14:59:17,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,019 DEBUG [hfile-prefetch-1733756355984 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6, with total blocks 2, and data blocks 1 2024-12-09T14:59:17,019 DEBUG [hfile-prefetch-1733756355984 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-09T14:59:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(7855): Warmup {ENCODED => 9710971f63f118d0433adb842186bd89, NAME => 'testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:59:17,019 DEBUG [hfile-prefetch-1733756355984 {}] bucket.BucketCache(2293): found block for file 46610c0fecde4828bfc867b0e62799e6 in the backing map. Acquiring read lock for offset 0 2024-12-09T14:59:17,019 DEBUG [hfile-prefetch-1733756355984 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6 has now been fully cached. 2024-12-09T14:59:17,019 DEBUG [hfile-prefetch-1733756355984 {}] hfile.PrefetchExecutor(121): Prefetch completed for 46610c0fecde4828bfc867b0e62799e6 2024-12-09T14:59:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:17,022 INFO [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,023 INFO [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9710971f63f118d0433adb842186bd89 columnFamilyName family 2024-12-09T14:59:17,023 DEBUG [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:17,027 DEBUG [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6 2024-12-09T14:59:17,027 INFO [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] regionserver.HStore(327): Store=9710971f63f118d0433adb842186bd89/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:59:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(1722): Closing 9710971f63f118d0433adb842186bd89, disabling compactions & flushes 2024-12-09T14:59:17,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. after waiting 0 ms 2024-12-09T14:59:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(1676): Region close journal for 9710971f63f118d0433adb842186bd89: Waiting for close lock at 1733756357027Disabling compacts and flushes for region at 1733756357027Disabling writes for close at 1733756357028 (+1 ms)Writing region close event to WAL at 1733756357029 (+1 ms)Closed at 1733756357029 2024-12-09T14:59:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34921 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-09T14:59:17,181 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,181 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-09T14:59:17,183 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 9710971f63f118d0433adb842186bd89, disabling compactions & flushes 2024-12-09T14:59:17,183 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,183 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,183 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. after waiting 0 ms 2024-12-09T14:59:17,183 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,188 DEBUG [StoreFileCloser-9710971f63f118d0433adb842186bd89-family-1 {}] bucket.BucketCache(768): Removing file 46610c0fecde4828bfc867b0e62799e6 for region 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,188 DEBUG [StoreFileCloser-9710971f63f118d0433adb842186bd89-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 46610c0fecde4828bfc867b0e62799e6, starting offset: 0, end offset: 9223372036854775807 2024-12-09T14:59:17,191 DEBUG [StoreFileCloser-9710971f63f118d0433adb842186bd89-family-1 {}] bucket.BucketCache(856): removed key 46610c0fecde4828bfc867b0e62799e6_0 from back map with offset lock 2094080 in the evict process 2024-12-09T14:59:17,195 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-09T14:59:17,196 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,196 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 9710971f63f118d0433adb842186bd89: Waiting for close lock at 1733756357182Running coprocessor pre-close hooks at 1733756357182Disabling compacts and flushes for region at 1733756357182Disabling writes for close at 1733756357183 (+1 ms)Writing region close event to WAL at 1733756357191 (+8 ms)Running coprocessor post-close hooks at 1733756357195 (+4 ms)Closed at 1733756357196 (+1 ms) 2024-12-09T14:59:17,197 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 9710971f63f118d0433adb842186bd89 move to 2491923345a5,38515,1733756352800 record at close sequenceid=6 2024-12-09T14:59:17,199 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,200 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9710971f63f118d0433adb842186bd89, regionState=CLOSED 2024-12-09T14:59:17,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9710971f63f118d0433adb842186bd89, server=2491923345a5,43285,1733756352941 because future has completed 2024-12-09T14:59:17,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T14:59:17,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 9710971f63f118d0433adb842186bd89, server=2491923345a5,43285,1733756352941 in 186 msec 2024-12-09T14:59:17,207 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733756355190, region=9710971f63f118d0433adb842186bd89, REOPEN/MOVE; state=CLOSED, location=2491923345a5,38515,1733756352800; forceNewPlan=false, retain=false 2024-12-09T14:59:17,217 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-09T14:59:17,217 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-09T14:59:17,217 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T14:59:17,217 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T14:59:17,220 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T14:59:17,220 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733756357217 2024-12-09T14:59:17,220 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-09T14:59:17,358 INFO [2491923345a5:34921 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T14:59:17,359 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9710971f63f118d0433adb842186bd89, regionState=OPENING, regionLocation=2491923345a5,38515,1733756352800 2024-12-09T14:59:17,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733756355190, region=9710971f63f118d0433adb842186bd89, REOPEN/MOVE because future has completed 2024-12-09T14:59:17,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9710971f63f118d0433adb842186bd89, server=2491923345a5,38515,1733756352800}] 2024-12-09T14:59:17,521 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,522 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 9710971f63f118d0433adb842186bd89, NAME => 'testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:59:17,523 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733756355190 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,523 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:17,523 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,523 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,524 INFO [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,525 INFO [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9710971f63f118d0433adb842186bd89 columnFamilyName family 2024-12-09T14:59:17,525 DEBUG [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:17,527 DEBUG [StoreFileOpener-9710971f63f118d0433adb842186bd89-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6, delay=934 ms 2024-12-09T14:59:17,528 DEBUG [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6 2024-12-09T14:59:17,528 INFO [StoreOpener-9710971f63f118d0433adb842186bd89-1 {}] regionserver.HStore(327): Store=9710971f63f118d0433adb842186bd89/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:59:17,529 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,529 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,530 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,530 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,530 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,533 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,534 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 9710971f63f118d0433adb842186bd89; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62288762, jitterRate=-0.07182511687278748}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:59:17,534 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,535 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 9710971f63f118d0433adb842186bd89: Running coprocessor pre-open hook at 1733756357523Writing region info on filesystem at 1733756357523Initializing all the Stores at 1733756357524 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756357524Cleaning up temporary data from old regions at 1733756357530 (+6 ms)Running coprocessor post-open hooks at 1733756357534 (+4 ms)Region opened successfully at 1733756357534 2024-12-09T14:59:17,535 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89., pid=9, masterSystemTime=1733756357519 2024-12-09T14:59:17,538 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,538 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,538 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9710971f63f118d0433adb842186bd89, regionState=OPEN, openSeqNum=10, regionLocation=2491923345a5,38515,1733756352800 2024-12-09T14:59:17,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9710971f63f118d0433adb842186bd89, server=2491923345a5,38515,1733756352800 because future has completed 2024-12-09T14:59:17,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-09T14:59:17,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 9710971f63f118d0433adb842186bd89, server=2491923345a5,38515,1733756352800 in 176 msec 2024-12-09T14:59:17,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733756355190, region=9710971f63f118d0433adb842186bd89, REOPEN/MOVE in 533 msec 2024-12-09T14:59:17,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T14:59:17,566 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T14:59:17,566 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:59:17,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:17,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:17,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T14:59:17,572 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T14:59:17,572 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=520236456, stopped=false 2024-12-09T14:59:17,573 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2491923345a5,34921,1733756352010 2024-12-09T14:59:17,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:17,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:17,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:17,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:17,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:17,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T14:59:17,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:17,596 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T14:59:17,597 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:59:17,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:17,597 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:17,597 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:17,598 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2491923345a5,38515,1733756352800' ***** 2024-12-09T14:59:17,598 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T14:59:17,598 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2491923345a5,43285,1733756352941' ***** 2024-12-09T14:59:17,598 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T14:59:17,599 INFO [RS:0;2491923345a5:38515 {}] bucket.BucketCache(1738): Disabling cache 2024-12-09T14:59:17,599 INFO [RS:1;2491923345a5:43285 {}] bucket.BucketCache(1738): Disabling cache 2024-12-09T14:59:17,599 INFO [RS:0;2491923345a5:38515 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/bucket.persistence 2024-12-09T14:59:17,599 INFO [RS:1;2491923345a5:43285 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/bucket.persistence 2024-12-09T14:59:17,600 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-09T14:59:17,600 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-09T14:59:17,600 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-09T14:59:17,601 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:17,601 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-09T14:59:17,601 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-09T14:59:17,601 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-09T14:59:17,601 INFO [RS:1;2491923345a5:43285 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-09T14:59:17,601 INFO [RS:0;2491923345a5:38515 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-09T14:59:17,601 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-09T14:59:17,601 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-09T14:59:17,611 DEBUG [RS:1;2491923345a5:43285 {}] bucket.BucketCache(1487): Thread RS:1;2491923345a5:43285 started persisting bucket cache to file 2024-12-09T14:59:17,611 DEBUG [RS:0;2491923345a5:38515 {}] bucket.BucketCache(1487): Thread RS:0;2491923345a5:38515 started persisting bucket cache to file 2024-12-09T14:59:17,611 DEBUG [RS:1;2491923345a5:43285 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T14:59:17,611 DEBUG [RS:0;2491923345a5:38515 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T14:59:17,611 DEBUG [RS:1;2491923345a5:43285 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T14:59:17,612 DEBUG [RS:0;2491923345a5:38515 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T14:59:17,612 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T14:59:17,614 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T14:59:17,614 DEBUG [RS:1;2491923345a5:43285 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T14:59:17,614 DEBUG [RS:1;2491923345a5:43285 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733756357611 2024-12-09T14:59:17,614 DEBUG [RS:0;2491923345a5:38515 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T14:59:17,614 DEBUG [RS:0;2491923345a5:38515 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733756357611 2024-12-09T14:59:17,614 DEBUG [RS:1;2491923345a5:43285 {}] bucket.BucketCache(1509): Thread RS:1;2491923345a5:43285 finished persisting bucket cache to file, renaming 2024-12-09T14:59:17,614 DEBUG [RS:0;2491923345a5:38515 {}] bucket.BucketCache(1509): Thread RS:0;2491923345a5:38515 finished persisting bucket cache to file, renaming 2024-12-09T14:59:17,615 WARN [RS:0;2491923345a5:38515 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-09T14:59:17,615 INFO [RS:1;2491923345a5:43285 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T14:59:17,615 INFO [RS:0;2491923345a5:38515 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T14:59:17,615 INFO [RS:0;2491923345a5:38515 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T14:59:17,615 INFO [RS:1;2491923345a5:43285 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T14:59:17,615 INFO [RS:1;2491923345a5:43285 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T14:59:17,615 INFO [RS:0;2491923345a5:38515 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T14:59:17,615 INFO [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(959): stopping server 2491923345a5,43285,1733756352941 2024-12-09T14:59:17,615 INFO [regionserver/2491923345a5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:59:17,615 INFO [regionserver/2491923345a5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:59:17,615 INFO [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(3091): Received CLOSE for 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,615 INFO [RS:1;2491923345a5:43285 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:59:17,616 INFO [RS:1;2491923345a5:43285 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2491923345a5:43285. 2024-12-09T14:59:17,616 DEBUG [RS:1;2491923345a5:43285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:59:17,616 DEBUG [RS:1;2491923345a5:43285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:17,616 INFO [RS:1;2491923345a5:43285 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T14:59:17,616 INFO [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(959): stopping server 2491923345a5,38515,1733756352800 2024-12-09T14:59:17,616 INFO [RS:1;2491923345a5:43285 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T14:59:17,616 INFO [RS:0;2491923345a5:38515 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:59:17,616 INFO [RS:1;2491923345a5:43285 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T14:59:17,616 INFO [RS:0;2491923345a5:38515 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2491923345a5:38515. 2024-12-09T14:59:17,616 INFO [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T14:59:17,616 DEBUG [RS:0;2491923345a5:38515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:59:17,616 DEBUG [RS:0;2491923345a5:38515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:17,616 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T14:59:17,616 INFO [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T14:59:17,616 INFO [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T14:59:17,617 DEBUG [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T14:59:17,617 DEBUG [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(1325): Online Regions={9710971f63f118d0433adb842186bd89=testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89.} 2024-12-09T14:59:17,617 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T14:59:17,617 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9710971f63f118d0433adb842186bd89, disabling compactions & flushes 2024-12-09T14:59:17,617 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T14:59:17,617 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,617 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T14:59:17,617 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,617 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T14:59:17,617 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. after waiting 0 ms 2024-12-09T14:59:17,617 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T14:59:17,617 DEBUG [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(1351): Waiting on 9710971f63f118d0433adb842186bd89 2024-12-09T14:59:17,617 DEBUG [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T14:59:17,617 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,617 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T14:59:17,617 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-09T14:59:17,618 DEBUG [StoreFileCloser-9710971f63f118d0433adb842186bd89-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/family/46610c0fecde4828bfc867b0e62799e6 2024-12-09T14:59:17,622 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnRegionMove1733756355190/9710971f63f118d0433adb842186bd89/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-09T14:59:17,623 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,623 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9710971f63f118d0433adb842186bd89: Waiting for close lock at 1733756357616Running coprocessor pre-close hooks at 1733756357617 (+1 ms)Disabling compacts and flushes for region at 1733756357617Disabling writes for close at 1733756357617Writing region close event to WAL at 1733756357618 (+1 ms)Running coprocessor post-close hooks at 1733756357623 (+5 ms)Closed at 1733756357623 2024-12-09T14:59:17,623 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89. 2024-12-09T14:59:17,626 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 070050f99e1f46c9a04e674cd5d06729 2024-12-09T14:59:17,642 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/info/070050f99e1f46c9a04e674cd5d06729 is 205, key is testBlockEvictionOnRegionMove1733756355190,,1733756355207.9710971f63f118d0433adb842186bd89./info:regioninfo/1733756357538/Put/seqid=0 2024-12-09T14:59:17,643 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/info/070050f99e1f46c9a04e674cd5d06729 2024-12-09T14:59:17,651 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f3093b5b4cea48c0aa45740a72e0996b 2024-12-09T14:59:17,663 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/ns/f3093b5b4cea48c0aa45740a72e0996b is 43, key is default/ns:d/1733756355008/Put/seqid=0 2024-12-09T14:59:17,663 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/ns/f3093b5b4cea48c0aa45740a72e0996b 2024-12-09T14:59:17,671 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9936fd51ccbb45899528750f3dd3503e 2024-12-09T14:59:17,685 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/table/9936fd51ccbb45899528750f3dd3503e is 78, key is testBlockEvictionOnRegionMove1733756355190/table:state/1733756355624/Put/seqid=0 2024-12-09T14:59:17,685 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/table/9936fd51ccbb45899528750f3dd3503e 2024-12-09T14:59:17,688 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/info/070050f99e1f46c9a04e674cd5d06729 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/070050f99e1f46c9a04e674cd5d06729 2024-12-09T14:59:17,690 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/070050f99e1f46c9a04e674cd5d06729, delay=947 ms 2024-12-09T14:59:17,691 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/070050f99e1f46c9a04e674cd5d06729, entries=15, sequenceid=15, filesize=7.6 K 2024-12-09T14:59:17,692 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/ns/f3093b5b4cea48c0aa45740a72e0996b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/ns/f3093b5b4cea48c0aa45740a72e0996b 2024-12-09T14:59:17,693 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/ns/f3093b5b4cea48c0aa45740a72e0996b, delay=908 ms 2024-12-09T14:59:17,694 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/ns/f3093b5b4cea48c0aa45740a72e0996b, entries=2, sequenceid=15, filesize=5.0 K 2024-12-09T14:59:17,695 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/table/9936fd51ccbb45899528750f3dd3503e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/table/9936fd51ccbb45899528750f3dd3503e 2024-12-09T14:59:17,696 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/table/9936fd51ccbb45899528750f3dd3503e, delay=939 ms 2024-12-09T14:59:17,697 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/table/9936fd51ccbb45899528750f3dd3503e, entries=2, sequenceid=15, filesize=5.3 K 2024-12-09T14:59:17,698 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=15, compaction requested=false 2024-12-09T14:59:17,698 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T14:59:17,699 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/070050f99e1f46c9a04e674cd5d06729 2024-12-09T14:59:17,701 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/ns/f3093b5b4cea48c0aa45740a72e0996b 2024-12-09T14:59:17,702 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/table/9936fd51ccbb45899528750f3dd3503e 2024-12-09T14:59:17,705 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-09T14:59:17,706 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T14:59:17,706 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T14:59:17,706 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733756357617Running coprocessor pre-close hooks at 1733756357617Disabling compacts and flushes for region at 1733756357617Disabling writes for close at 1733756357617Obtaining lock to block concurrent updates at 1733756357617Preparing flush snapshotting stores in 1588230740 at 1733756357617Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733756357618 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733756357618Flushing 1588230740/info: creating writer at 1733756357618Flushing 1588230740/info: appending metadata at 1733756357641 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733756357641Flushing 1588230740/ns: creating writer at 1733756357645 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733756357662 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733756357662Flushing 1588230740/table: creating writer at 1733756357666 (+4 ms)Flushing 1588230740/table: appending metadata at 1733756357684 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733756357684Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24e2aba: reopening flushed file at 1733756357688 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c4541d4: reopening flushed file at 1733756357692 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@83c65f5: reopening flushed file at 1733756357695 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=15, compaction requested=false at 1733756357698 (+3 ms)Writing region close event to WAL at 1733756357702 (+4 ms)Running coprocessor post-close hooks at 1733756357706 (+4 ms)Closed at 1733756357706 2024-12-09T14:59:17,707 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T14:59:17,818 INFO [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(976): stopping server 2491923345a5,43285,1733756352941; all regions closed. 2024-12-09T14:59:17,818 INFO [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(976): stopping server 2491923345a5,38515,1733756352800; all regions closed. 2024-12-09T14:59:17,825 DEBUG [RS:0;2491923345a5:38515 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs 2024-12-09T14:59:17,825 DEBUG [RS:1;2491923345a5:43285 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs 2024-12-09T14:59:17,826 INFO [RS:1;2491923345a5:43285 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2491923345a5%2C43285%2C1733756352941.meta:.meta(num 1733756354861) 2024-12-09T14:59:17,826 INFO [RS:0;2491923345a5:38515 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2491923345a5%2C38515%2C1733756352800:(num 1733756354467) 2024-12-09T14:59:17,826 DEBUG [RS:0;2491923345a5:38515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:17,826 INFO [RS:0;2491923345a5:38515 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:59:17,826 INFO [RS:0;2491923345a5:38515 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:59:17,826 INFO [RS:0;2491923345a5:38515 {}] hbase.ChoreService(370): Chore service for: regionserver/2491923345a5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T14:59:17,826 INFO [RS:0;2491923345a5:38515 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T14:59:17,826 INFO [regionserver/2491923345a5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:59:17,826 INFO [RS:0;2491923345a5:38515 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T14:59:17,826 INFO [RS:0;2491923345a5:38515 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T14:59:17,826 INFO [RS:0;2491923345a5:38515 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:59:17,827 INFO [RS:0;2491923345a5:38515 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38515 2024-12-09T14:59:17,827 DEBUG [RS:1;2491923345a5:43285 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs 2024-12-09T14:59:17,827 INFO [RS:1;2491923345a5:43285 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2491923345a5%2C43285%2C1733756352941:(num 1733756354467) 2024-12-09T14:59:17,827 DEBUG [RS:1;2491923345a5:43285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:17,827 INFO [RS:1;2491923345a5:43285 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:59:17,827 INFO [RS:1;2491923345a5:43285 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:59:17,827 INFO [RS:1;2491923345a5:43285 {}] hbase.ChoreService(370): Chore service for: regionserver/2491923345a5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T14:59:17,828 INFO [RS:1;2491923345a5:43285 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:59:17,828 INFO [regionserver/2491923345a5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:59:17,828 INFO [RS:1;2491923345a5:43285 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43285 2024-12-09T14:59:17,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2491923345a5,43285,1733756352941 2024-12-09T14:59:17,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:59:17,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2491923345a5,38515,1733756352800 2024-12-09T14:59:17,862 INFO [RS:1;2491923345a5:43285 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:59:17,862 INFO [RS:0;2491923345a5:38515 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:59:17,870 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2491923345a5,43285,1733756352941] 2024-12-09T14:59:17,878 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2491923345a5,43285,1733756352941 already deleted, retry=false 2024-12-09T14:59:17,879 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2491923345a5,43285,1733756352941 expired; onlineServers=1 2024-12-09T14:59:17,879 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2491923345a5,38515,1733756352800] 2024-12-09T14:59:17,886 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2491923345a5,38515,1733756352800 already deleted, retry=false 2024-12-09T14:59:17,886 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2491923345a5,38515,1733756352800 expired; onlineServers=0 2024-12-09T14:59:17,886 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2491923345a5,34921,1733756352010' ***** 2024-12-09T14:59:17,886 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T14:59:17,887 INFO [M:0;2491923345a5:34921 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:59:17,887 INFO [M:0;2491923345a5:34921 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:59:17,887 DEBUG [M:0;2491923345a5:34921 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T14:59:17,887 DEBUG [M:0;2491923345a5:34921 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T14:59:17,887 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T14:59:17,887 DEBUG [master/2491923345a5:0:becomeActiveMaster-HFileCleaner.small.0-1733756354061 {}] cleaner.HFileCleaner(306): Exit Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.small.0-1733756354061,5,FailOnTimeoutGroup] 2024-12-09T14:59:17,887 DEBUG [master/2491923345a5:0:becomeActiveMaster-HFileCleaner.large.0-1733756354059 {}] cleaner.HFileCleaner(306): Exit Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.large.0-1733756354059,5,FailOnTimeoutGroup] 2024-12-09T14:59:17,887 INFO [M:0;2491923345a5:34921 {}] hbase.ChoreService(370): Chore service for: master/2491923345a5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T14:59:17,887 INFO [M:0;2491923345a5:34921 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:59:17,887 DEBUG [M:0;2491923345a5:34921 {}] master.HMaster(1795): Stopping service threads 2024-12-09T14:59:17,888 INFO [M:0;2491923345a5:34921 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T14:59:17,888 INFO [M:0;2491923345a5:34921 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T14:59:17,888 INFO [M:0;2491923345a5:34921 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T14:59:17,888 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T14:59:17,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T14:59:17,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:17,895 DEBUG [M:0;2491923345a5:34921 {}] zookeeper.ZKUtil(347): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T14:59:17,895 WARN [M:0;2491923345a5:34921 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T14:59:17,896 INFO [M:0;2491923345a5:34921 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/.lastflushedseqids 2024-12-09T14:59:17,905 INFO [M:0;2491923345a5:34921 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T14:59:17,905 INFO [M:0;2491923345a5:34921 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T14:59:17,905 DEBUG [M:0;2491923345a5:34921 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T14:59:17,905 INFO [M:0;2491923345a5:34921 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:17,905 DEBUG [M:0;2491923345a5:34921 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:17,905 DEBUG [M:0;2491923345a5:34921 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T14:59:17,905 DEBUG [M:0;2491923345a5:34921 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:17,906 INFO [M:0;2491923345a5:34921 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-12-09T14:59:17,914 DEBUG [M:0;2491923345a5:34921 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6743b4756ad54f168a131c08a0d8171a 2024-12-09T14:59:17,927 DEBUG [M:0;2491923345a5:34921 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6743b4756ad54f168a131c08a0d8171a is 82, key is hbase:meta,,1/info:regioninfo/1733756354923/Put/seqid=0 2024-12-09T14:59:17,927 INFO [M:0;2491923345a5:34921 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6743b4756ad54f168a131c08a0d8171a 2024-12-09T14:59:17,934 DEBUG [M:0;2491923345a5:34921 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 61892815dd3442e38d8649473fde20e1 2024-12-09T14:59:17,945 DEBUG [M:0;2491923345a5:34921 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/61892815dd3442e38d8649473fde20e1 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733756355629/Put/seqid=0 2024-12-09T14:59:17,946 INFO [M:0;2491923345a5:34921 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/61892815dd3442e38d8649473fde20e1 2024-12-09T14:59:17,953 DEBUG [M:0;2491923345a5:34921 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2b97501c48bd4343a9c50abd040191dc 2024-12-09T14:59:17,963 DEBUG [M:0;2491923345a5:34921 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2b97501c48bd4343a9c50abd040191dc is 69, key is 2491923345a5,38515,1733756352800/rs:state/1733756354179/Put/seqid=0 2024-12-09T14:59:17,963 INFO [M:0;2491923345a5:34921 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2b97501c48bd4343a9c50abd040191dc 2024-12-09T14:59:17,965 DEBUG [M:0;2491923345a5:34921 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6743b4756ad54f168a131c08a0d8171a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6743b4756ad54f168a131c08a0d8171a 2024-12-09T14:59:17,966 INFO [M:0;2491923345a5:34921 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6743b4756ad54f168a131c08a0d8171a, entries=8, sequenceid=84, filesize=5.5 K 2024-12-09T14:59:17,967 DEBUG [M:0;2491923345a5:34921 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/61892815dd3442e38d8649473fde20e1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/61892815dd3442e38d8649473fde20e1 2024-12-09T14:59:17,968 INFO [M:0;2491923345a5:34921 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/61892815dd3442e38d8649473fde20e1, entries=9, sequenceid=84, filesize=6.5 K 2024-12-09T14:59:17,969 DEBUG [M:0;2491923345a5:34921 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2b97501c48bd4343a9c50abd040191dc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2b97501c48bd4343a9c50abd040191dc 2024-12-09T14:59:17,970 INFO [M:0;2491923345a5:34921 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2b97501c48bd4343a9c50abd040191dc, entries=2, sequenceid=84, filesize=5.1 K 2024-12-09T14:59:17,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:17,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43285-0x1000b24338f0002, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:17,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:17,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1000b24338f0001, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:17,971 INFO [RS:0;2491923345a5:38515 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:59:17,971 INFO [RS:1;2491923345a5:43285 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:59:17,971 INFO [RS:0;2491923345a5:38515 {}] regionserver.HRegionServer(1031): Exiting; stopping=2491923345a5,38515,1733756352800; zookeeper connection closed. 2024-12-09T14:59:17,971 INFO [RS:1;2491923345a5:43285 {}] regionserver.HRegionServer(1031): Exiting; stopping=2491923345a5,43285,1733756352941; zookeeper connection closed. 2024-12-09T14:59:17,971 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6eb6ce53 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6eb6ce53 2024-12-09T14:59:17,971 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3533ae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3533ae 2024-12-09T14:59:17,971 INFO [M:0;2491923345a5:34921 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 66ms, sequenceid=84, compaction requested=false 2024-12-09T14:59:17,972 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-09T14:59:17,973 INFO [M:0;2491923345a5:34921 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:17,973 DEBUG [M:0;2491923345a5:34921 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733756357905Disabling compacts and flushes for region at 1733756357905Disabling writes for close at 1733756357905Obtaining lock to block concurrent updates at 1733756357906 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733756357906Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733756357906Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733756357906Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733756357906Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733756357926 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733756357926Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733756357929 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733756357945 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733756357945Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733756357948 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733756357962 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733756357962Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ed281db: reopening flushed file at 1733756357965 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@543341d9: reopening flushed file at 1733756357967 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19fc3cbe: reopening flushed file at 1733756357968 (+1 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 66ms, sequenceid=84, compaction requested=false at 1733756357971 (+3 ms)Writing region close event to WAL at 1733756357973 (+2 ms)Closed at 1733756357973 2024-12-09T14:59:17,974 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:59:17,974 INFO [M:0;2491923345a5:34921 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T14:59:17,974 INFO [M:0;2491923345a5:34921 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34921 2024-12-09T14:59:17,974 INFO [M:0;2491923345a5:34921 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:59:18,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:18,087 INFO [M:0;2491923345a5:34921 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:59:18,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34921-0x1000b24338f0000, quorum=127.0.0.1:50273, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:18,095 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T14:59:18,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-09T14:59:18,115 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:18,121 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/2491923345a5:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2491923345a5:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733756357690 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733756355984 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2491923345a5:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733756357693 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733756357527 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/2491923345a5:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 309), ProcessCount=11 (was 11), AvailableMemoryMB=7789 (was 8125) 2024-12-09T14:59:18,126 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=7789 2024-12-09T14:59:18,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/hadoop-log-dir so I do NOT create it in target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa 2024-12-09T14:59:18,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/hadoop-tmp-dir so I do NOT create it in target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa 2024-12-09T14:59:18,127 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:18,129 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/cluster_416d669b-7d08-e9b5-6ba6-58bf390a92c4, deleteOnExit=true 2024-12-09T14:59:18,131 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/cluster_416d669b-7d08-e9b5-6ba6-58bf390a92c4/zookeeper_0, clientPort=56049, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/cluster_416d669b-7d08-e9b5-6ba6-58bf390a92c4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/cluster_416d669b-7d08-e9b5-6ba6-58bf390a92c4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T14:59:18,132 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56049 2024-12-09T14:59:18,142 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a with version=8 2024-12-09T14:59:18,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/hbase-staging 2024-12-09T14:59:18,145 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2491923345a5:0 server-side Connection retries=45 2024-12-09T14:59:18,145 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:18,145 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:18,145 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:59:18,146 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:18,146 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:59:18,146 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T14:59:18,146 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:59:18,147 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39975 2024-12-09T14:59:18,149 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39975 connecting to ZooKeeper ensemble=127.0.0.1:56049 2024-12-09T14:59:18,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:399750x0, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:59:18,194 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39975-0x1000b244c8f0000 connected 2024-12-09T14:59:18,294 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:18,295 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:18,298 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:18,299 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a, hbase.cluster.distributed=false 2024-12-09T14:59:18,302 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:59:18,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39975 2024-12-09T14:59:18,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39975 2024-12-09T14:59:18,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39975 2024-12-09T14:59:18,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39975 2024-12-09T14:59:18,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39975 2024-12-09T14:59:18,322 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2491923345a5:0 server-side Connection retries=45 2024-12-09T14:59:18,322 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:18,322 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:18,322 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:59:18,322 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:18,322 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:59:18,322 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T14:59:18,322 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:59:18,323 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40093 2024-12-09T14:59:18,324 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40093 connecting to ZooKeeper ensemble=127.0.0.1:56049 2024-12-09T14:59:18,324 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:18,325 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:18,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400930x0, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:59:18,335 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40093-0x1000b244c8f0001 connected 2024-12-09T14:59:18,335 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:18,336 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T14:59:18,337 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.cache 2024-12-09T14:59:18,337 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-09T14:59:18,337 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-09T14:59:18,337 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-09T14:59:18,337 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-09T14:59:18,337 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-09T14:59:18,337 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-09T14:59:18,338 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T14:59:18,338 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-09T14:59:18,338 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.persistence is complete. 2024-12-09T14:59:18,339 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T14:59:18,340 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:59:18,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40093 2024-12-09T14:59:18,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40093 2024-12-09T14:59:18,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40093 2024-12-09T14:59:18,342 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40093 2024-12-09T14:59:18,342 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40093 2024-12-09T14:59:18,357 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2491923345a5:0 server-side Connection retries=45 2024-12-09T14:59:18,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:18,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:18,358 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:59:18,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:18,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:59:18,358 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T14:59:18,358 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:59:18,359 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34487 2024-12-09T14:59:18,360 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34487 connecting to ZooKeeper ensemble=127.0.0.1:56049 2024-12-09T14:59:18,360 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:18,360 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:18,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:344870x0, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:59:18,372 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34487-0x1000b244c8f0002 connected 2024-12-09T14:59:18,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:18,372 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T14:59:18,373 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.cache 2024-12-09T14:59:18,373 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-09T14:59:18,373 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-09T14:59:18,373 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-09T14:59:18,374 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-09T14:59:18,374 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-09T14:59:18,374 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-09T14:59:18,374 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.persistence is complete. 2024-12-09T14:59:18,374 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-09T14:59:18,374 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T14:59:18,375 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T14:59:18,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:59:18,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34487 2024-12-09T14:59:18,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34487 2024-12-09T14:59:18,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34487 2024-12-09T14:59:18,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34487 2024-12-09T14:59:18,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34487 2024-12-09T14:59:18,393 DEBUG [M:0;2491923345a5:39975 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2491923345a5:39975 2024-12-09T14:59:18,393 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2491923345a5,39975,1733756358144 2024-12-09T14:59:18,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:18,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:18,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:18,406 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2491923345a5,39975,1733756358144 2024-12-09T14:59:18,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T14:59:18,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T14:59:18,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,414 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T14:59:18,414 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2491923345a5,39975,1733756358144 from backup master directory 2024-12-09T14:59:18,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:18,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2491923345a5,39975,1733756358144 2024-12-09T14:59:18,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:18,421 WARN [master/2491923345a5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:59:18,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:59:18,421 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2491923345a5,39975,1733756358144 2024-12-09T14:59:18,423 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/hbase.id] with ID: bde89f51-148e-4bf2-9954-46191346eee0 2024-12-09T14:59:18,423 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/.tmp/hbase.id 2024-12-09T14:59:18,426 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/hbase.id] 2024-12-09T14:59:18,460 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:18,460 INFO [master/2491923345a5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T14:59:18,462 INFO [master/2491923345a5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-09T14:59:18,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,475 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T14:59:18,476 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T14:59:18,476 INFO [master/2491923345a5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:59:18,494 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store 2024-12-09T14:59:18,501 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:18,501 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T14:59:18,502 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:18,502 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:18,502 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T14:59:18,502 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:18,502 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:18,502 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733756358501Disabling compacts and flushes for region at 1733756358501Disabling writes for close at 1733756358502 (+1 ms)Writing region close event to WAL at 1733756358502Closed at 1733756358502 2024-12-09T14:59:18,502 WARN [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/.initializing 2024-12-09T14:59:18,502 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/WALs/2491923345a5,39975,1733756358144 2024-12-09T14:59:18,509 INFO [master/2491923345a5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:59:18,514 INFO [master/2491923345a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2491923345a5%2C39975%2C1733756358144, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/WALs/2491923345a5,39975,1733756358144, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/oldWALs, maxLogs=10 2024-12-09T14:59:18,535 INFO [master/2491923345a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/WALs/2491923345a5,39975,1733756358144/2491923345a5%2C39975%2C1733756358144.1733756358514 2024-12-09T14:59:18,535 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T14:59:18,536 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:59:18,536 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:18,536 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,536 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,537 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,540 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T14:59:18,540 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:18,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:18,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,544 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T14:59:18,544 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:18,544 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:59:18,544 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T14:59:18,547 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:18,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:59:18,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T14:59:18,550 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:18,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:59:18,551 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,551 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,551 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,552 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,552 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,552 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:59:18,552 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:59:18,557 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:59:18,557 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60096829, jitterRate=-0.10448746383190155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:59:18,558 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733756358536Initializing all the Stores at 1733756358536Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756358536Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756358536Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756358536Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756358536Cleaning up temporary data from old regions at 1733756358552 (+16 ms)Region opened successfully at 1733756358558 (+6 ms) 2024-12-09T14:59:18,559 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T14:59:18,562 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158b8935, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2491923345a5/172.17.0.2:0 2024-12-09T14:59:18,563 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T14:59:18,563 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T14:59:18,563 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T14:59:18,563 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T14:59:18,564 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T14:59:18,564 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-09T14:59:18,564 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T14:59:18,566 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T14:59:18,567 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T14:59:18,576 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T14:59:18,577 INFO [master/2491923345a5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T14:59:18,578 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T14:59:18,588 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T14:59:18,588 INFO [master/2491923345a5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T14:59:18,589 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T14:59:18,596 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T14:59:18,597 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T14:59:18,604 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T14:59:18,608 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T14:59:18,618 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T14:59:18,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:18,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:18,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:18,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,630 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2491923345a5,39975,1733756358144, sessionid=0x1000b244c8f0000, setting cluster-up flag (Was=false) 2024-12-09T14:59:18,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,671 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T14:59:18,673 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2491923345a5,39975,1733756358144 2024-12-09T14:59:18,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:18,713 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T14:59:18,714 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2491923345a5,39975,1733756358144 2024-12-09T14:59:18,715 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T14:59:18,716 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T14:59:18,717 INFO [master/2491923345a5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T14:59:18,717 INFO [master/2491923345a5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T14:59:18,717 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2491923345a5,39975,1733756358144 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T14:59:18,719 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:59:18,719 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:59:18,719 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:59:18,719 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:59:18,719 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2491923345a5:0, corePoolSize=10, maxPoolSize=10 2024-12-09T14:59:18,720 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,720 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2491923345a5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:59:18,720 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,720 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733756388720 2024-12-09T14:59:18,721 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T14:59:18,721 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T14:59:18,721 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T14:59:18,721 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T14:59:18,721 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T14:59:18,721 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T14:59:18,721 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,721 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:59:18,722 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T14:59:18,722 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T14:59:18,722 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T14:59:18,722 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T14:59:18,723 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:18,723 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T14:59:18,723 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T14:59:18,723 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.large.0-1733756358723,5,FailOnTimeoutGroup] 2024-12-09T14:59:18,723 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T14:59:18,723 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.small.0-1733756358723,5,FailOnTimeoutGroup] 2024-12-09T14:59:18,723 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,724 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T14:59:18,724 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,724 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,739 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T14:59:18,739 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a 2024-12-09T14:59:18,744 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:18,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-09T14:59:18,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T14:59:18,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:18,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:18,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-09T14:59:18,750 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T14:59:18,751 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:18,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:18,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-09T14:59:18,753 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T14:59:18,753 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:18,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:18,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-09T14:59:18,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T14:59:18,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:18,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:18,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T14:59:18,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740 2024-12-09T14:59:18,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740 2024-12-09T14:59:18,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T14:59:18,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T14:59:18,760 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:59:18,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T14:59:18,766 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:59:18,767 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59936743, jitterRate=-0.10687293112277985}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:59:18,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733756358745Initializing all the Stores at 1733756358745Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756358745Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756358745Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756358745Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756358745Cleaning up temporary data from old regions at 1733756358760 (+15 ms)Region opened successfully at 1733756358768 (+8 ms) 2024-12-09T14:59:18,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T14:59:18,768 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T14:59:18,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T14:59:18,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T14:59:18,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T14:59:18,769 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T14:59:18,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733756358768Disabling compacts and flushes for region at 1733756358768Disabling writes for close at 1733756358768Writing region close event to WAL at 1733756358769 (+1 ms)Closed at 1733756358769 2024-12-09T14:59:18,770 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:59:18,770 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T14:59:18,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T14:59:18,772 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T14:59:18,773 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T14:59:18,785 INFO [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(746): ClusterId : bde89f51-148e-4bf2-9954-46191346eee0 2024-12-09T14:59:18,785 INFO [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(746): ClusterId : bde89f51-148e-4bf2-9954-46191346eee0 2024-12-09T14:59:18,785 DEBUG [RS:0;2491923345a5:40093 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T14:59:18,785 DEBUG [RS:1;2491923345a5:34487 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T14:59:18,814 DEBUG [RS:1;2491923345a5:34487 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T14:59:18,814 DEBUG [RS:0;2491923345a5:40093 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T14:59:18,814 DEBUG [RS:1;2491923345a5:34487 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T14:59:18,814 DEBUG [RS:0;2491923345a5:40093 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T14:59:18,823 DEBUG [RS:1;2491923345a5:34487 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T14:59:18,823 DEBUG [RS:0;2491923345a5:40093 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T14:59:18,823 DEBUG [RS:0;2491923345a5:40093 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a9b4ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2491923345a5/172.17.0.2:0 2024-12-09T14:59:18,824 DEBUG [RS:1;2491923345a5:34487 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@680782a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2491923345a5/172.17.0.2:0 2024-12-09T14:59:18,839 DEBUG [RS:1;2491923345a5:34487 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2491923345a5:34487 2024-12-09T14:59:18,839 INFO [RS:1;2491923345a5:34487 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T14:59:18,839 INFO [RS:1;2491923345a5:34487 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T14:59:18,839 DEBUG [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T14:59:18,840 INFO [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(2659): reportForDuty to master=2491923345a5,39975,1733756358144 with port=34487, startcode=1733756358357 2024-12-09T14:59:18,840 DEBUG [RS:1;2491923345a5:34487 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T14:59:18,841 DEBUG [RS:0;2491923345a5:40093 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2491923345a5:40093 2024-12-09T14:59:18,841 INFO [RS:0;2491923345a5:40093 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T14:59:18,841 INFO [RS:0;2491923345a5:40093 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T14:59:18,841 DEBUG [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T14:59:18,842 INFO [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(2659): reportForDuty to master=2491923345a5,39975,1733756358144 with port=40093, startcode=1733756358321 2024-12-09T14:59:18,842 DEBUG [RS:0;2491923345a5:40093 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T14:59:18,843 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60139, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T14:59:18,844 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39975 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2491923345a5,34487,1733756358357 2024-12-09T14:59:18,844 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39975 {}] master.ServerManager(517): Registering regionserver=2491923345a5,34487,1733756358357 2024-12-09T14:59:18,845 DEBUG [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a 2024-12-09T14:59:18,845 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49393, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T14:59:18,845 DEBUG [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-09T14:59:18,845 DEBUG [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T14:59:18,846 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39975 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2491923345a5,40093,1733756358321 2024-12-09T14:59:18,846 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39975 {}] master.ServerManager(517): Registering regionserver=2491923345a5,40093,1733756358321 2024-12-09T14:59:18,848 DEBUG [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a 2024-12-09T14:59:18,848 DEBUG [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-09T14:59:18,848 DEBUG [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T14:59:18,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:59:18,885 DEBUG [RS:1;2491923345a5:34487 {}] zookeeper.ZKUtil(111): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2491923345a5,34487,1733756358357 2024-12-09T14:59:18,885 WARN [RS:1;2491923345a5:34487 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:59:18,885 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2491923345a5,34487,1733756358357] 2024-12-09T14:59:18,885 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2491923345a5,40093,1733756358321] 2024-12-09T14:59:18,885 INFO [RS:1;2491923345a5:34487 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:59:18,885 DEBUG [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,34487,1733756358357 2024-12-09T14:59:18,885 DEBUG [RS:0;2491923345a5:40093 {}] zookeeper.ZKUtil(111): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2491923345a5,40093,1733756358321 2024-12-09T14:59:18,885 WARN [RS:0;2491923345a5:40093 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:59:18,886 INFO [RS:0;2491923345a5:40093 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:59:18,886 DEBUG [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,40093,1733756358321 2024-12-09T14:59:18,890 INFO [RS:0;2491923345a5:40093 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T14:59:18,890 INFO [RS:1;2491923345a5:34487 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T14:59:18,892 INFO [RS:0;2491923345a5:40093 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T14:59:18,893 INFO [RS:0;2491923345a5:40093 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:59:18,893 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,893 INFO [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T14:59:18,894 INFO [RS:0;2491923345a5:40093 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T14:59:18,894 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,894 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,894 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,894 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,894 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,894 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,895 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:59:18,895 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,895 INFO [RS:1;2491923345a5:34487 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T14:59:18,895 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,895 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,895 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,895 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,895 INFO [RS:1;2491923345a5:34487 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:59:18,895 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,895 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,895 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:59:18,895 DEBUG [RS:0;2491923345a5:40093 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:59:18,895 INFO [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T14:59:18,896 INFO [RS:1;2491923345a5:34487 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T14:59:18,896 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,896 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,896 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,896 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,896 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,896 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,897 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:59:18,897 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,897 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,897 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,897 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,897 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,897 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,897 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:18,897 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,897 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,897 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:59:18,897 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,897 DEBUG [RS:1;2491923345a5:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:59:18,897 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,897 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,40093,1733756358321-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:59:18,903 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,903 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,903 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,903 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,903 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,903 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,34487,1733756358357-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:59:18,913 INFO [RS:0;2491923345a5:40093 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T14:59:18,913 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,40093,1733756358321-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,913 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,913 INFO [RS:0;2491923345a5:40093 {}] regionserver.Replication(171): 2491923345a5,40093,1733756358321 started 2024-12-09T14:59:18,923 WARN [2491923345a5:39975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T14:59:18,923 INFO [RS:1;2491923345a5:34487 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T14:59:18,923 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,34487,1733756358357-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,924 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,924 INFO [RS:1;2491923345a5:34487 {}] regionserver.Replication(171): 2491923345a5,34487,1733756358357 started 2024-12-09T14:59:18,925 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,925 INFO [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(1482): Serving as 2491923345a5,40093,1733756358321, RpcServer on 2491923345a5/172.17.0.2:40093, sessionid=0x1000b244c8f0001 2024-12-09T14:59:18,925 DEBUG [RS:0;2491923345a5:40093 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T14:59:18,925 DEBUG [RS:0;2491923345a5:40093 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2491923345a5,40093,1733756358321 2024-12-09T14:59:18,925 DEBUG [RS:0;2491923345a5:40093 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,40093,1733756358321' 2024-12-09T14:59:18,925 DEBUG [RS:0;2491923345a5:40093 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T14:59:18,926 DEBUG [RS:0;2491923345a5:40093 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T14:59:18,927 DEBUG [RS:0;2491923345a5:40093 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T14:59:18,927 DEBUG [RS:0;2491923345a5:40093 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T14:59:18,927 DEBUG [RS:0;2491923345a5:40093 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2491923345a5,40093,1733756358321 2024-12-09T14:59:18,927 DEBUG [RS:0;2491923345a5:40093 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,40093,1733756358321' 2024-12-09T14:59:18,927 DEBUG [RS:0;2491923345a5:40093 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T14:59:18,927 DEBUG [RS:0;2491923345a5:40093 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T14:59:18,928 DEBUG [RS:0;2491923345a5:40093 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T14:59:18,928 INFO [RS:0;2491923345a5:40093 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T14:59:18,928 INFO [RS:0;2491923345a5:40093 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T14:59:18,939 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:18,939 INFO [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(1482): Serving as 2491923345a5,34487,1733756358357, RpcServer on 2491923345a5/172.17.0.2:34487, sessionid=0x1000b244c8f0002 2024-12-09T14:59:18,939 DEBUG [RS:1;2491923345a5:34487 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T14:59:18,939 DEBUG [RS:1;2491923345a5:34487 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2491923345a5,34487,1733756358357 2024-12-09T14:59:18,939 DEBUG [RS:1;2491923345a5:34487 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,34487,1733756358357' 2024-12-09T14:59:18,940 DEBUG [RS:1;2491923345a5:34487 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T14:59:18,940 DEBUG [RS:1;2491923345a5:34487 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T14:59:18,941 DEBUG [RS:1;2491923345a5:34487 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T14:59:18,941 DEBUG [RS:1;2491923345a5:34487 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T14:59:18,941 DEBUG [RS:1;2491923345a5:34487 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2491923345a5,34487,1733756358357 2024-12-09T14:59:18,941 DEBUG [RS:1;2491923345a5:34487 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,34487,1733756358357' 2024-12-09T14:59:18,941 DEBUG [RS:1;2491923345a5:34487 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T14:59:18,941 DEBUG [RS:1;2491923345a5:34487 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T14:59:18,942 DEBUG [RS:1;2491923345a5:34487 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T14:59:18,942 INFO [RS:1;2491923345a5:34487 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T14:59:18,942 INFO [RS:1;2491923345a5:34487 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T14:59:19,029 INFO [RS:0;2491923345a5:40093 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:59:19,030 INFO [RS:0;2491923345a5:40093 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2491923345a5%2C40093%2C1733756358321, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,40093,1733756358321, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs, maxLogs=55 2024-12-09T14:59:19,043 INFO [RS:1;2491923345a5:34487 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:59:19,043 INFO [RS:1;2491923345a5:34487 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2491923345a5%2C34487%2C1733756358357, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,34487,1733756358357, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs, maxLogs=55 2024-12-09T14:59:19,052 INFO [RS:0;2491923345a5:40093 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,40093,1733756358321/2491923345a5%2C40093%2C1733756358321.1733756359032 2024-12-09T14:59:19,053 DEBUG [RS:0;2491923345a5:40093 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T14:59:19,059 INFO [RS:1;2491923345a5:34487 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,34487,1733756358357/2491923345a5%2C34487%2C1733756358357.1733756359045 2024-12-09T14:59:19,059 DEBUG [RS:1;2491923345a5:34487 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T14:59:19,173 DEBUG [2491923345a5:39975 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-09T14:59:19,174 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(204): Hosts are {2491923345a5=0} racks are {/default-rack=0} 2024-12-09T14:59:19,174 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T14:59:19,174 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T14:59:19,174 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T14:59:19,174 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T14:59:19,174 INFO [2491923345a5:39975 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T14:59:19,174 INFO [2491923345a5:39975 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T14:59:19,175 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T14:59:19,175 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2491923345a5,34487,1733756358357 2024-12-09T14:59:19,176 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2491923345a5,34487,1733756358357, state=OPENING 2024-12-09T14:59:19,205 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T14:59:19,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:19,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:19,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:19,214 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T14:59:19,214 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:19,215 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:19,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2491923345a5,34487,1733756358357}] 2024-12-09T14:59:19,215 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:19,371 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:59:19,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56721, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:59:19,378 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T14:59:19,378 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:59:19,379 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T14:59:19,380 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2491923345a5%2C34487%2C1733756358357.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,34487,1733756358357, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs, maxLogs=55 2024-12-09T14:59:19,395 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,34487,1733756358357/2491923345a5%2C34487%2C1733756358357.meta.1733756359380.meta 2024-12-09T14:59:19,396 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T14:59:19,396 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:59:19,396 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T14:59:19,396 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T14:59:19,396 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T14:59:19,396 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T14:59:19,396 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:19,397 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T14:59:19,397 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T14:59:19,397 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-09T14:59:19,398 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T14:59:19,398 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:19,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:19,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-09T14:59:19,399 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T14:59:19,399 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:19,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:19,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-09T14:59:19,399 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T14:59:19,399 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:19,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:19,400 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-09T14:59:19,400 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T14:59:19,400 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:19,400 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:19,400 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T14:59:19,401 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740 2024-12-09T14:59:19,401 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740 2024-12-09T14:59:19,401 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T14:59:19,401 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T14:59:19,401 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:59:19,402 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T14:59:19,402 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60248823, jitterRate=-0.10222257673740387}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:59:19,402 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T14:59:19,403 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733756359397Writing region info on filesystem at 1733756359397Initializing all the Stores at 1733756359397Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756359397Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756359397Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756359397Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756359397Cleaning up temporary data from old regions at 1733756359401 (+4 ms)Running coprocessor post-open hooks at 1733756359402 (+1 ms)Region opened successfully at 1733756359403 (+1 ms) 2024-12-09T14:59:19,404 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733756359370 2024-12-09T14:59:19,406 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T14:59:19,406 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T14:59:19,407 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2491923345a5,34487,1733756358357 2024-12-09T14:59:19,407 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2491923345a5,34487,1733756358357, state=OPEN 2024-12-09T14:59:19,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:59:19,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:59:19,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:59:19,430 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2491923345a5,34487,1733756358357 2024-12-09T14:59:19,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:19,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:19,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:19,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T14:59:19,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2491923345a5,34487,1733756358357 in 215 msec 2024-12-09T14:59:19,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T14:59:19,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 661 msec 2024-12-09T14:59:19,434 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:59:19,434 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T14:59:19,437 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T14:59:19,437 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2491923345a5,34487,1733756358357, seqNum=-1] 2024-12-09T14:59:19,437 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:59:19,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35275, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:59:19,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 728 msec 2024-12-09T14:59:19,444 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733756359444, completionTime=-1 2024-12-09T14:59:19,444 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-09T14:59:19,444 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T14:59:19,447 INFO [master/2491923345a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T14:59:19,447 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733756419447 2024-12-09T14:59:19,447 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733756479447 2024-12-09T14:59:19,447 INFO [master/2491923345a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-09T14:59:19,447 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,39975,1733756358144-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:19,447 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,39975,1733756358144-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:19,447 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,39975,1733756358144-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:19,448 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2491923345a5:39975, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:19,448 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:19,448 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:19,450 DEBUG [master/2491923345a5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T14:59:19,454 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.033sec 2024-12-09T14:59:19,454 INFO [master/2491923345a5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T14:59:19,454 INFO [master/2491923345a5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T14:59:19,454 INFO [master/2491923345a5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T14:59:19,455 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T14:59:19,455 INFO [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T14:59:19,455 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,39975,1733756358144-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:59:19,455 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,39975,1733756358144-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T14:59:19,456 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T14:59:19,456 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T14:59:19,456 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,39975,1733756358144-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:19,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3076c0d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:59:19,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2491923345a5,39975,-1 for getting cluster id 2024-12-09T14:59:19,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T14:59:19,487 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bde89f51-148e-4bf2-9954-46191346eee0' 2024-12-09T14:59:19,487 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T14:59:19,488 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bde89f51-148e-4bf2-9954-46191346eee0" 2024-12-09T14:59:19,488 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5976a4e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:59:19,488 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2491923345a5,39975,-1] 2024-12-09T14:59:19,488 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T14:59:19,489 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:19,490 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T14:59:19,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:59:19,491 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T14:59:19,493 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2491923345a5,34487,1733756358357, seqNum=-1] 2024-12-09T14:59:19,493 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:59:19,495 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:59:19,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2491923345a5,39975,1733756358144 2024-12-09T14:59:19,498 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T14:59:19,499 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2491923345a5,39975,1733756358144 2024-12-09T14:59:19,499 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@20442acc 2024-12-09T14:59:19,500 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T14:59:19,502 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T14:59:19,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733756359498', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T14:59:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39975 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733756359498 2024-12-09T14:59:19,505 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733756359498 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T14:59:19,505 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:19,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733756359498" procId is: 4 2024-12-09T14:59:19,506 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733756359498 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T14:59:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:59:19,514 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733756359498-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 81254ceb5b41c7a9b3cb279f574da4d8, NAME => 'testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733756359498', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a 2024-12-09T14:59:19,520 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733756359498-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:19,520 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733756359498-pool-0 {}] regionserver.HRegion(1722): Closing 81254ceb5b41c7a9b3cb279f574da4d8, disabling compactions & flushes 2024-12-09T14:59:19,520 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733756359498-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:19,520 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733756359498-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:19,520 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733756359498-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. after waiting 0 ms 2024-12-09T14:59:19,520 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733756359498-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:19,520 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733756359498-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:19,520 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733756359498-pool-0 {}] regionserver.HRegion(1676): Region close journal for 81254ceb5b41c7a9b3cb279f574da4d8: Waiting for close lock at 1733756359520Disabling compacts and flushes for region at 1733756359520Disabling writes for close at 1733756359520Writing region close event to WAL at 1733756359520Closed at 1733756359520 2024-12-09T14:59:19,521 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733756359498 execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T14:59:19,521 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733756359521"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733756359521"}]},"ts":"1733756359521"} 2024-12-09T14:59:19,524 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T14:59:19,525 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733756359498 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T14:59:19,525 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733756359498","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733756359525"}]},"ts":"1733756359525"} 2024-12-09T14:59:19,527 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733756359498, state=ENABLING in hbase:meta 2024-12-09T14:59:19,527 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2491923345a5=0} racks are {/default-rack=0} 2024-12-09T14:59:19,528 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T14:59:19,528 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T14:59:19,528 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T14:59:19,528 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T14:59:19,528 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T14:59:19,528 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T14:59:19,528 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T14:59:19,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733756359498, region=81254ceb5b41c7a9b3cb279f574da4d8, ASSIGN}] 2024-12-09T14:59:19,529 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733756359498, region=81254ceb5b41c7a9b3cb279f574da4d8, ASSIGN 2024-12-09T14:59:19,530 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733756359498, region=81254ceb5b41c7a9b3cb279f574da4d8, ASSIGN; state=OFFLINE, location=2491923345a5,34487,1733756358357; forceNewPlan=false, retain=false 2024-12-09T14:59:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:59:19,681 INFO [2491923345a5:39975 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T14:59:19,681 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=81254ceb5b41c7a9b3cb279f574da4d8, regionState=OPENING, regionLocation=2491923345a5,34487,1733756358357 2024-12-09T14:59:19,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733756359498, region=81254ceb5b41c7a9b3cb279f574da4d8, ASSIGN because future has completed 2024-12-09T14:59:19,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81254ceb5b41c7a9b3cb279f574da4d8, server=2491923345a5,34487,1733756358357}] 2024-12-09T14:59:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:59:19,846 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:19,847 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 81254ceb5b41c7a9b3cb279f574da4d8, NAME => 'testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:59:19,847 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733756359498 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:19,847 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:19,847 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:19,847 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:19,848 INFO [StoreOpener-81254ceb5b41c7a9b3cb279f574da4d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:19,852 INFO [StoreOpener-81254ceb5b41c7a9b3cb279f574da4d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81254ceb5b41c7a9b3cb279f574da4d8 columnFamilyName family 2024-12-09T14:59:19,852 DEBUG [StoreOpener-81254ceb5b41c7a9b3cb279f574da4d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:19,852 INFO [StoreOpener-81254ceb5b41c7a9b3cb279f574da4d8-1 {}] regionserver.HStore(327): Store=81254ceb5b41c7a9b3cb279f574da4d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:59:19,853 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:19,853 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:19,854 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:19,854 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:19,854 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:19,854 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:19,862 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:59:19,862 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 81254ceb5b41c7a9b3cb279f574da4d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70564629, jitterRate=0.051494911313056946}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:59:19,862 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:19,863 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 81254ceb5b41c7a9b3cb279f574da4d8: Running coprocessor pre-open hook at 1733756359848Writing region info on filesystem at 1733756359848Initializing all the Stores at 1733756359848Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756359848Cleaning up temporary data from old regions at 1733756359854 (+6 ms)Running coprocessor post-open hooks at 1733756359863 (+9 ms)Region opened successfully at 1733756359863 2024-12-09T14:59:19,864 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8., pid=6, masterSystemTime=1733756359840 2024-12-09T14:59:19,867 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:19,867 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:19,867 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=81254ceb5b41c7a9b3cb279f574da4d8, regionState=OPEN, openSeqNum=2, regionLocation=2491923345a5,34487,1733756358357 2024-12-09T14:59:19,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81254ceb5b41c7a9b3cb279f574da4d8, server=2491923345a5,34487,1733756358357 because future has completed 2024-12-09T14:59:19,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T14:59:19,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 81254ceb5b41c7a9b3cb279f574da4d8, server=2491923345a5,34487,1733756358357 in 184 msec 2024-12-09T14:59:19,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T14:59:19,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733756359498, region=81254ceb5b41c7a9b3cb279f574da4d8, ASSIGN in 345 msec 2024-12-09T14:59:19,876 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733756359498 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T14:59:19,876 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733756359498","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733756359876"}]},"ts":"1733756359876"} 2024-12-09T14:59:19,878 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733756359498, state=ENABLED in hbase:meta 2024-12-09T14:59:19,879 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733756359498 execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T14:59:19,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733756359498 in 376 msec 2024-12-09T14:59:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:59:20,137 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733756359498 completed 2024-12-09T14:59:20,137 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733756359498 get assigned. Timeout = 60000ms 2024-12-09T14:59:20,138 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T14:59:20,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733756359498 assigned to meta. Checking AM states. 2024-12-09T14:59:20,144 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T14:59:20,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733756359498 assigned. 2024-12-09T14:59:20,149 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733756359498', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8., hostname=2491923345a5,34487,1733756358357, seqNum=2] 2024-12-09T14:59:20,155 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 81254ceb5b41c7a9b3cb279f574da4d8 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-09T14:59:20,162 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d9eacab60a2f45f39e3ad1151819319f 2024-12-09T14:59:20,175 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/.tmp/family/d9eacab60a2f45f39e3ad1151819319f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-09T14:59:20,175 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/.tmp/family/d9eacab60a2f45f39e3ad1151819319f 2024-12-09T14:59:20,178 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/.tmp/family/d9eacab60a2f45f39e3ad1151819319f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/family/d9eacab60a2f45f39e3ad1151819319f 2024-12-09T14:59:20,179 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/family/d9eacab60a2f45f39e3ad1151819319f, delay=997 ms 2024-12-09T14:59:20,180 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/family/d9eacab60a2f45f39e3ad1151819319f, entries=2, sequenceid=6, filesize=4.9 K 2024-12-09T14:59:20,180 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 81254ceb5b41c7a9b3cb279f574da4d8 in 25ms, sequenceid=6, compaction requested=false 2024-12-09T14:59:20,180 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733756359498' 2024-12-09T14:59:20,181 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 81254ceb5b41c7a9b3cb279f574da4d8: 2024-12-09T14:59:20,398 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T14:59:21,177 DEBUG [hfile-prefetch-1733756357527 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d9eacab60a2f45f39e3ad1151819319f_0. 2024-12-09T14:59:21,178 DEBUG [hfile-prefetch-1733756357527 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d9eacab60a2f45f39e3ad1151819319f_117. 2024-12-09T14:59:21,178 DEBUG [hfile-prefetch-1733756357527 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/family/d9eacab60a2f45f39e3ad1151819319f, with total blocks 2, and data blocks 1 2024-12-09T14:59:21,178 DEBUG [hfile-prefetch-1733756357527 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-09T14:59:21,178 DEBUG [hfile-prefetch-1733756357527 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/family/d9eacab60a2f45f39e3ad1151819319f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-09T14:59:21,178 DEBUG [hfile-prefetch-1733756357527 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/family/d9eacab60a2f45f39e3ad1151819319f. Will sleep 100ms and try the verification again. 2024-12-09T14:59:21,182 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;2491923345a5:34487,5,FailOnTimeoutGroup] 2024-12-09T14:59:21,182 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2491923345a5,34487,1733756358357' ***** 2024-12-09T14:59:21,182 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-09T14:59:21,182 INFO [RS:1;2491923345a5:34487 {}] bucket.BucketCache(1738): Disabling cache 2024-12-09T14:59:21,203 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T14:59:21,206 INFO [regionserver/2491923345a5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:59:21,244 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-09T14:59:21,244 INFO [RS:1;2491923345a5:34487 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.persistence 2024-12-09T14:59:21,244 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-09T14:59:21,244 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-09T14:59:21,244 INFO [RS:1;2491923345a5:34487 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-09T14:59:21,245 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-09T14:59:21,255 DEBUG [RS:1;2491923345a5:34487 {}] bucket.BucketCache(1487): Thread RS:1;2491923345a5:34487 started persisting bucket cache to file 2024-12-09T14:59:21,255 DEBUG [RS:1;2491923345a5:34487 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T14:59:21,255 DEBUG [RS:1;2491923345a5:34487 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T14:59:21,263 DEBUG [RS:1;2491923345a5:34487 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-09T14:59:21,263 DEBUG [RS:1;2491923345a5:34487 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733756361255 2024-12-09T14:59:21,263 DEBUG [RS:1;2491923345a5:34487 {}] bucket.BucketCache(1509): Thread RS:1;2491923345a5:34487 finished persisting bucket cache to file, renaming 2024-12-09T14:59:21,264 INFO [RS:1;2491923345a5:34487 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T14:59:21,264 INFO [RS:1;2491923345a5:34487 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T14:59:21,264 INFO [RS:1;2491923345a5:34487 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T14:59:21,264 INFO [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(3091): Received CLOSE for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:21,264 INFO [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(959): stopping server 2491923345a5,34487,1733756358357 2024-12-09T14:59:21,264 INFO [RS:1;2491923345a5:34487 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:59:21,264 INFO [RS:1;2491923345a5:34487 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2491923345a5:34487. 2024-12-09T14:59:21,264 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T14:59:21,264 DEBUG [RS:1;2491923345a5:34487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:59:21,264 DEBUG [RS:1;2491923345a5:34487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:21,264 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 81254ceb5b41c7a9b3cb279f574da4d8, disabling compactions & flushes 2024-12-09T14:59:21,264 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:21,264 INFO [RS:1;2491923345a5:34487 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T14:59:21,265 INFO [RS:1;2491923345a5:34487 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T14:59:21,265 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:21,265 INFO [RS:1;2491923345a5:34487 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T14:59:21,265 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. after waiting 0 ms 2024-12-09T14:59:21,265 INFO [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T14:59:21,265 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:21,265 INFO [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-09T14:59:21,265 DEBUG [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 81254ceb5b41c7a9b3cb279f574da4d8=testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8.} 2024-12-09T14:59:21,265 DEBUG [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:21,265 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T14:59:21,265 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T14:59:21,265 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T14:59:21,265 DEBUG [StoreFileCloser-81254ceb5b41c7a9b3cb279f574da4d8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/family/d9eacab60a2f45f39e3ad1151819319f 2024-12-09T14:59:21,265 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T14:59:21,265 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T14:59:21,266 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T14:59:21,266 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-09T14:59:21,267 WARN [hfile-prefetch-1733756357527 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/family/d9eacab60a2f45f39e3ad1151819319f, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-09T14:59:21,268 DEBUG [hfile-prefetch-1733756357527 {}] hfile.PrefetchExecutor(121): Prefetch completed for d9eacab60a2f45f39e3ad1151819319f 2024-12-09T14:59:21,275 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-09T14:59:21,275 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:21,275 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 81254ceb5b41c7a9b3cb279f574da4d8: Waiting for close lock at 1733756361264Running coprocessor pre-close hooks at 1733756361264Disabling compacts and flushes for region at 1733756361264Disabling writes for close at 1733756361265 (+1 ms)Writing region close event to WAL at 1733756361266 (+1 ms)Running coprocessor post-close hooks at 1733756361275 (+9 ms)Closed at 1733756361275 2024-12-09T14:59:21,275 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:21,278 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f37dbe1cee74437693532b99bdf57efa 2024-12-09T14:59:21,290 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/info/f37dbe1cee74437693532b99bdf57efa is 209, key is testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8./info:regioninfo/1733756359867/Put/seqid=0 2024-12-09T14:59:21,290 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/info/f37dbe1cee74437693532b99bdf57efa 2024-12-09T14:59:21,297 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d40f7c95b1a6492c91342f6dee25dc74 2024-12-09T14:59:21,307 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/ns/d40f7c95b1a6492c91342f6dee25dc74 is 43, key is default/ns:d/1733756359439/Put/seqid=0 2024-12-09T14:59:21,308 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/ns/d40f7c95b1a6492c91342f6dee25dc74 2024-12-09T14:59:21,315 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cb60e9ebf9cf46149d6a24756082cde3 2024-12-09T14:59:21,326 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/table/cb60e9ebf9cf46149d6a24756082cde3 is 80, key is testBlockEvictionOnGracefulStop1733756359498/table:state/1733756359876/Put/seqid=0 2024-12-09T14:59:21,326 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/table/cb60e9ebf9cf46149d6a24756082cde3 2024-12-09T14:59:21,328 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/info/f37dbe1cee74437693532b99bdf57efa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/f37dbe1cee74437693532b99bdf57efa 2024-12-09T14:59:21,329 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/f37dbe1cee74437693532b99bdf57efa, delay=981 ms 2024-12-09T14:59:21,330 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/f37dbe1cee74437693532b99bdf57efa, entries=10, sequenceid=11, filesize=7.0 K 2024-12-09T14:59:21,331 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/ns/d40f7c95b1a6492c91342f6dee25dc74 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/ns/d40f7c95b1a6492c91342f6dee25dc74 2024-12-09T14:59:21,331 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/ns/d40f7c95b1a6492c91342f6dee25dc74, delay=919 ms 2024-12-09T14:59:21,332 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/ns/d40f7c95b1a6492c91342f6dee25dc74, entries=2, sequenceid=11, filesize=5.0 K 2024-12-09T14:59:21,332 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/table/cb60e9ebf9cf46149d6a24756082cde3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/table/cb60e9ebf9cf46149d6a24756082cde3 2024-12-09T14:59:21,333 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/table/cb60e9ebf9cf46149d6a24756082cde3, delay=983 ms 2024-12-09T14:59:21,334 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/table/cb60e9ebf9cf46149d6a24756082cde3, entries=2, sequenceid=11, filesize=5.3 K 2024-12-09T14:59:21,335 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 69ms, sequenceid=11, compaction requested=false 2024-12-09T14:59:21,335 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/f37dbe1cee74437693532b99bdf57efa 2024-12-09T14:59:21,336 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/ns/d40f7c95b1a6492c91342f6dee25dc74 2024-12-09T14:59:21,336 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/table/cb60e9ebf9cf46149d6a24756082cde3 2024-12-09T14:59:21,340 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-09T14:59:21,340 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-09T14:59:21,340 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-09T14:59:21,340 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T14:59:21,340 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T14:59:21,340 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T14:59:21,340 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T14:59:21,340 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733756361265Running coprocessor pre-close hooks at 1733756361265Disabling compacts and flushes for region at 1733756361265Disabling writes for close at 1733756361265Obtaining lock to block concurrent updates at 1733756361266 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733756361266Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733756361266Flushing stores of hbase:meta,,1.1588230740 at 1733756361267 (+1 ms)Flushing 1588230740/info: creating writer at 1733756361267Flushing 1588230740/info: appending metadata at 1733756361289 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733756361289Flushing 1588230740/ns: creating writer at 1733756361292 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733756361307 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733756361307Flushing 1588230740/table: creating writer at 1733756361309 (+2 ms)Flushing 1588230740/table: appending metadata at 1733756361325 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733756361325Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5eb41db5: reopening flushed file at 1733756361328 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bbc445e: reopening flushed file at 1733756361330 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2472d811: reopening flushed file at 1733756361332 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 69ms, sequenceid=11, compaction requested=false at 1733756361335 (+3 ms)Writing region close event to WAL at 1733756361337 (+2 ms)Running coprocessor post-close hooks at 1733756361340 (+3 ms)Closed at 1733756361340 2024-12-09T14:59:21,340 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T14:59:21,342 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T14:59:21,342 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733756361340 2024-12-09T14:59:21,342 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-09T14:59:21,465 INFO [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(976): stopping server 2491923345a5,34487,1733756358357; all regions closed. 2024-12-09T14:59:21,469 DEBUG [RS:1;2491923345a5:34487 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs 2024-12-09T14:59:21,469 INFO [RS:1;2491923345a5:34487 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2491923345a5%2C34487%2C1733756358357.meta:.meta(num 1733756359380) 2024-12-09T14:59:21,472 DEBUG [RS:1;2491923345a5:34487 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs 2024-12-09T14:59:21,472 INFO [RS:1;2491923345a5:34487 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2491923345a5%2C34487%2C1733756358357:(num 1733756359045) 2024-12-09T14:59:21,472 DEBUG [RS:1;2491923345a5:34487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:21,473 INFO [RS:1;2491923345a5:34487 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:59:21,473 INFO [RS:1;2491923345a5:34487 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:59:21,473 INFO [RS:1;2491923345a5:34487 {}] hbase.ChoreService(370): Chore service for: regionserver/2491923345a5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T14:59:21,473 INFO [RS:1;2491923345a5:34487 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:59:21,473 INFO [regionserver/2491923345a5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:59:21,473 INFO [RS:1;2491923345a5:34487 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34487 2024-12-09T14:59:21,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2491923345a5,34487,1733756358357 2024-12-09T14:59:21,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:59:21,509 INFO [RS:1;2491923345a5:34487 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:59:21,510 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2491923345a5,34487,1733756358357] 2024-12-09T14:59:21,529 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2491923345a5,34487,1733756358357 already deleted, retry=false 2024-12-09T14:59:21,530 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2491923345a5,34487,1733756358357 on 2491923345a5,39975,1733756358144 2024-12-09T14:59:21,537 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2491923345a5,34487,1733756358357, splitWal=true, meta=true 2024-12-09T14:59:21,540 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2491923345a5,34487,1733756358357, splitWal=true, meta=true 2024-12-09T14:59:21,540 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 2491923345a5,34487,1733756358357 (carryingMeta=true) 2491923345a5,34487,1733756358357/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2af2ea2c[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T14:59:21,540 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 2491923345a5,34487,1733756358357, splitWal=true, meta=true, isMeta: true 2024-12-09T14:59:21,541 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,34487,1733756358357-splitting 2024-12-09T14:59:21,541 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,34487,1733756358357-splitting dir is empty, no logs to split. 2024-12-09T14:59:21,541 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2491923345a5,34487,1733756358357 WAL count=0, meta=true 2024-12-09T14:59:21,543 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,34487,1733756358357-splitting dir is empty, no logs to split. 2024-12-09T14:59:21,543 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2491923345a5,34487,1733756358357 WAL count=0, meta=true 2024-12-09T14:59:21,543 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 2491923345a5,34487,1733756358357 WAL splitting is done? wals=0, meta=true 2024-12-09T14:59:21,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T14:59:21,545 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T14:59:21,545 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T14:59:21,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:21,621 INFO [RS:1;2491923345a5:34487 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:59:21,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x1000b244c8f0002, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:21,621 INFO [RS:1;2491923345a5:34487 {}] regionserver.HRegionServer(1031): Exiting; stopping=2491923345a5,34487,1733756358357; zookeeper connection closed. 2024-12-09T14:59:21,622 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@523efb9c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@523efb9c 2024-12-09T14:59:21,696 DEBUG [2491923345a5:39975 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T14:59:21,696 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2491923345a5,40093,1733756358321 2024-12-09T14:59:21,697 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2491923345a5,40093,1733756358321, state=OPENING 2024-12-09T14:59:21,703 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2491923345a5:0 server-side Connection retries=45 2024-12-09T14:59:21,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:21,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:21,703 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:59:21,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:59:21,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:59:21,703 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T14:59:21,703 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:59:21,704 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41967 2024-12-09T14:59:21,705 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41967 connecting to ZooKeeper ensemble=127.0.0.1:56049 2024-12-09T14:59:21,705 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:21,705 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:21,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:59:21,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:59:21,726 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T14:59:21,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:21,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:21,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2491923345a5,40093,1733756358321}] 2024-12-09T14:59:21,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419670x0, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:59:21,735 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-09T14:59:21,735 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:419670x0, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-09T14:59:21,735 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41967-0x1000b244c8f0003 connected 2024-12-09T14:59:21,735 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T14:59:21,737 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.cache 2024-12-09T14:59:21,737 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-09T14:59:21,737 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-09T14:59:21,737 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-09T14:59:21,737 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-09T14:59:21,737 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-09T14:59:21,738 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T14:59:21,738 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41967-0x1000b244c8f0003, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T14:59:21,739 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-09T14:59:21,739 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-09T14:59:21,740 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41967-0x1000b244c8f0003, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:59:21,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41967 2024-12-09T14:59:21,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41967 2024-12-09T14:59:21,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41967 2024-12-09T14:59:21,747 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41967 2024-12-09T14:59:21,747 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-09T14:59:21,747 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41967 2024-12-09T14:59:21,748 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-09T14:59:21,748 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-09T14:59:21,748 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.persistence is complete. 2024-12-09T14:59:21,749 INFO [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer(746): ClusterId : bde89f51-148e-4bf2-9954-46191346eee0 2024-12-09T14:59:21,749 DEBUG [RS:2;2491923345a5:41967 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T14:59:21,760 DEBUG [RS:2;2491923345a5:41967 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T14:59:21,760 DEBUG [RS:2;2491923345a5:41967 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T14:59:21,772 DEBUG [RS:2;2491923345a5:41967 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T14:59:21,772 DEBUG [RS:2;2491923345a5:41967 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33969b2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2491923345a5/172.17.0.2:0 2024-12-09T14:59:21,783 DEBUG [RS:2;2491923345a5:41967 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2491923345a5:41967 2024-12-09T14:59:21,783 INFO [RS:2;2491923345a5:41967 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T14:59:21,783 INFO [RS:2;2491923345a5:41967 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T14:59:21,783 DEBUG [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T14:59:21,784 INFO [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer(2659): reportForDuty to master=2491923345a5,39975,1733756358144 with port=41967, startcode=1733756361702 2024-12-09T14:59:21,784 DEBUG [RS:2;2491923345a5:41967 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T14:59:21,786 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33831, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T14:59:21,786 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39975 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2491923345a5,41967,1733756361702 2024-12-09T14:59:21,786 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39975 {}] master.ServerManager(517): Registering regionserver=2491923345a5,41967,1733756361702 2024-12-09T14:59:21,788 DEBUG [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a 2024-12-09T14:59:21,788 DEBUG [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-09T14:59:21,788 DEBUG [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T14:59:21,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:59:21,796 DEBUG [RS:2;2491923345a5:41967 {}] zookeeper.ZKUtil(111): regionserver:41967-0x1000b244c8f0003, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2491923345a5,41967,1733756361702 2024-12-09T14:59:21,796 WARN [RS:2;2491923345a5:41967 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:59:21,797 INFO [RS:2;2491923345a5:41967 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:59:21,797 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2491923345a5,41967,1733756361702] 2024-12-09T14:59:21,797 DEBUG [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,41967,1733756361702 2024-12-09T14:59:21,801 INFO [RS:2;2491923345a5:41967 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T14:59:21,803 INFO [RS:2;2491923345a5:41967 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T14:59:21,803 INFO [RS:2;2491923345a5:41967 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:59:21,803 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:21,803 INFO [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T14:59:21,804 INFO [RS:2;2491923345a5:41967 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T14:59:21,804 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:21,804 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:21,804 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:21,804 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:21,804 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:21,804 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:21,804 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:59:21,804 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:21,804 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:21,804 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:21,804 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:21,804 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:21,805 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:59:21,805 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:59:21,805 DEBUG [RS:2;2491923345a5:41967 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:59:21,805 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:21,805 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:21,805 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:21,805 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:21,805 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:21,805 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,41967,1733756361702-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:59:21,819 INFO [RS:2;2491923345a5:41967 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T14:59:21,820 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,41967,1733756361702-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:21,820 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:21,820 INFO [RS:2;2491923345a5:41967 {}] regionserver.Replication(171): 2491923345a5,41967,1733756361702 started 2024-12-09T14:59:21,832 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:59:21,832 INFO [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer(1482): Serving as 2491923345a5,41967,1733756361702, RpcServer on 2491923345a5/172.17.0.2:41967, sessionid=0x1000b244c8f0003 2024-12-09T14:59:21,832 DEBUG [RS:2;2491923345a5:41967 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T14:59:21,832 DEBUG [RS:2;2491923345a5:41967 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2491923345a5,41967,1733756361702 2024-12-09T14:59:21,832 DEBUG [RS:2;2491923345a5:41967 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,41967,1733756361702' 2024-12-09T14:59:21,832 DEBUG [RS:2;2491923345a5:41967 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T14:59:21,833 DEBUG [RS:2;2491923345a5:41967 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T14:59:21,834 DEBUG [RS:2;2491923345a5:41967 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T14:59:21,834 DEBUG [RS:2;2491923345a5:41967 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T14:59:21,834 DEBUG [RS:2;2491923345a5:41967 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2491923345a5,41967,1733756361702 2024-12-09T14:59:21,834 DEBUG [RS:2;2491923345a5:41967 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,41967,1733756361702' 2024-12-09T14:59:21,834 DEBUG [RS:2;2491923345a5:41967 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T14:59:21,834 DEBUG [RS:2;2491923345a5:41967 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T14:59:21,835 DEBUG [RS:2;2491923345a5:41967 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T14:59:21,835 INFO [RS:2;2491923345a5:41967 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T14:59:21,835 INFO [RS:2;2491923345a5:41967 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T14:59:21,879 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:59:21,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40777, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:59:21,882 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T14:59:21,882 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:59:21,883 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T14:59:21,884 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2491923345a5%2C40093%2C1733756358321.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,40093,1733756358321, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs, maxLogs=55 2024-12-09T14:59:21,898 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,40093,1733756358321/2491923345a5%2C40093%2C1733756358321.meta.1733756361884.meta 2024-12-09T14:59:21,898 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T14:59:21,899 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:59:21,899 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T14:59:21,899 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T14:59:21,899 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T14:59:21,899 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T14:59:21,899 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:21,899 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T14:59:21,900 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T14:59:21,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-09T14:59:21,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T14:59:21,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:21,902 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/f37dbe1cee74437693532b99bdf57efa, delay=912 ms 2024-12-09T14:59:21,904 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/f37dbe1cee74437693532b99bdf57efa 2024-12-09T14:59:21,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:21,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-09T14:59:21,905 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T14:59:21,905 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:21,907 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/ns/d40f7c95b1a6492c91342f6dee25dc74, delay=939 ms 2024-12-09T14:59:21,909 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/ns/d40f7c95b1a6492c91342f6dee25dc74 2024-12-09T14:59:21,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:21,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-09T14:59:21,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T14:59:21,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:21,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:21,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-09T14:59:21,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T14:59:21,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:21,913 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/table/cb60e9ebf9cf46149d6a24756082cde3, delay=925 ms 2024-12-09T14:59:21,914 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/table/cb60e9ebf9cf46149d6a24756082cde3 2024-12-09T14:59:21,914 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:59:21,914 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T14:59:21,915 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740 2024-12-09T14:59:21,915 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740 2024-12-09T14:59:21,916 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T14:59:21,916 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T14:59:21,916 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:59:21,917 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T14:59:21,917 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62803800, jitterRate=-0.06415045261383057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:59:21,918 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T14:59:21,918 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733756361900Writing region info on filesystem at 1733756361900Initializing all the Stores at 1733756361900Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756361900Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756361900Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756361900Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733756361900Cleaning up temporary data from old regions at 1733756361916 (+16 ms)Running coprocessor post-open hooks at 1733756361918 (+2 ms)Region opened successfully at 1733756361918 2024-12-09T14:59:21,919 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733756361879 2024-12-09T14:59:21,922 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T14:59:21,922 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T14:59:21,922 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=2491923345a5,40093,1733756358321 2024-12-09T14:59:21,923 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2491923345a5,40093,1733756358321, state=OPEN 2024-12-09T14:59:21,932 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39975 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2491923345a5,40093,1733756358321, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T14:59:21,935 INFO [RS:2;2491923345a5:41967 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:59:21,936 INFO [RS:2;2491923345a5:41967 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2491923345a5%2C41967%2C1733756361702, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,41967,1733756361702, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs, maxLogs=55 2024-12-09T14:59:21,952 INFO [RS:2;2491923345a5:41967 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,41967,1733756361702/2491923345a5%2C41967%2C1733756361702.1733756361937 2024-12-09T14:59:21,952 DEBUG [RS:2;2491923345a5:41967 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T14:59:21,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:59:21,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41967-0x1000b244c8f0003, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:59:21,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:59:21,954 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2491923345a5,40093,1733756358321 2024-12-09T14:59:21,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:21,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:21,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:59:21,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-09T14:59:21,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2491923345a5,40093,1733756358321 in 228 msec 2024-12-09T14:59:21,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T14:59:21,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 413 msec 2024-12-09T14:59:21,959 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 2491923345a5,34487,1733756358357 had 2 regions 2024-12-09T14:59:21,960 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2491923345a5,34487,1733756358357, splitWal=true, meta=true, isMeta: false 2024-12-09T14:59:21,960 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,34487,1733756358357-splitting dir is empty, no logs to split. 2024-12-09T14:59:21,960 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2491923345a5,34487,1733756358357 WAL count=0, meta=false 2024-12-09T14:59:21,961 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/WALs/2491923345a5,34487,1733756358357-splitting dir is empty, no logs to split. 2024-12-09T14:59:21,961 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2491923345a5,34487,1733756358357 WAL count=0, meta=false 2024-12-09T14:59:21,961 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 2491923345a5,34487,1733756358357 WAL splitting is done? wals=0, meta=false 2024-12-09T14:59:21,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733756359498, region=81254ceb5b41c7a9b3cb279f574da4d8, ASSIGN}] 2024-12-09T14:59:21,963 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733756359498, region=81254ceb5b41c7a9b3cb279f574da4d8, ASSIGN 2024-12-09T14:59:21,964 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733756359498, region=81254ceb5b41c7a9b3cb279f574da4d8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T14:59:22,115 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(204): Hosts are {2491923345a5=0} racks are {/default-rack=0} 2024-12-09T14:59:22,115 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T14:59:22,115 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T14:59:22,115 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T14:59:22,115 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T14:59:22,115 INFO [2491923345a5:39975 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T14:59:22,115 INFO [2491923345a5:39975 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T14:59:22,115 DEBUG [2491923345a5:39975 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T14:59:22,116 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=81254ceb5b41c7a9b3cb279f574da4d8, regionState=OPENING, regionLocation=2491923345a5,41967,1733756361702 2024-12-09T14:59:22,118 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2491923345a5:34487 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2491923345a5/172.17.0.2:34487 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:59:22,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2491923345a5,34487,1733756358357, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2491923345a5,34487,1733756358357, seqNum=-1, error=java.net.ConnectException: Call to address=2491923345a5:34487 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2491923345a5/172.17.0.2:34487 2024-12-09T14:59:22,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2491923345a5,34487,1733756358357, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-09T14:59:22,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2491923345a5,34487,1733756358357, seqNum=-1 from cache 2024-12-09T14:59:22,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 2491923345a5:34487 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2491923345a5/172.17.0.2:34487 2024-12-09T14:59:22,224 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T14:59:22,224 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2491923345a5,40093,1733756358321, seqNum=-1] 2024-12-09T14:59:22,225 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:59:22,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59361, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:59:22,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733756359498, region=81254ceb5b41c7a9b3cb279f574da4d8, ASSIGN because future has completed 2024-12-09T14:59:22,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81254ceb5b41c7a9b3cb279f574da4d8, server=2491923345a5,41967,1733756361702}] 2024-12-09T14:59:22,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T14:59:22,333 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T14:59:22,333 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:59:22,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:22,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:22,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T14:59:22,334 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T14:59:22,334 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1154003689, stopped=false 2024-12-09T14:59:22,334 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2491923345a5,39975,1733756358144 2024-12-09T14:59:22,386 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:59:22,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35365, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:59:22,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:22,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:22,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41967-0x1000b244c8f0003, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:59:22,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:22,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41967-0x1000b244c8f0003, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:22,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:22,394 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T14:59:22,394 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T14:59:22,394 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:22,394 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:59:22,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:22,395 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:22,395 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:22,395 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 81254ceb5b41c7a9b3cb279f574da4d8, NAME => 'testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:59:22,395 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41967-0x1000b244c8f0003, quorum=127.0.0.1:56049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:59:22,395 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2491923345a5,40093,1733756358321' ***** 2024-12-09T14:59:22,395 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T14:59:22,395 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2491923345a5,41967,1733756361702' ***** 2024-12-09T14:59:22,395 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T14:59:22,395 INFO [RS:2;2491923345a5:41967 {}] bucket.BucketCache(1738): Disabling cache 2024-12-09T14:59:22,396 INFO [RS:2;2491923345a5:41967 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.persistence 2024-12-09T14:59:22,396 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-09T14:59:22,396 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-09T14:59:22,396 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-09T14:59:22,396 INFO [RS:2;2491923345a5:41967 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-09T14:59:22,396 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T14:59:22,396 INFO [RS:0;2491923345a5:40093 {}] bucket.BucketCache(1738): Disabling cache 2024-12-09T14:59:22,396 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733756359498 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:22,396 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:59:22,396 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-09T14:59:22,396 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-09T14:59:22,396 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-09T14:59:22,396 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:22,397 INFO [RS:0;2491923345a5:40093 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/7aabe7fc-5267-eb68-c213-76cf80581aaa/bucket.persistence 2024-12-09T14:59:22,397 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:22,397 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-09T14:59:22,397 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T14:59:22,397 DEBUG [RS:2;2491923345a5:41967 {}] bucket.BucketCache(1487): Thread RS:2;2491923345a5:41967 started persisting bucket cache to file 2024-12-09T14:59:22,397 INFO [RS:0;2491923345a5:40093 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-09T14:59:22,397 DEBUG [RS:2;2491923345a5:41967 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T14:59:22,397 DEBUG [RS:2;2491923345a5:41967 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T14:59:22,397 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-09T14:59:22,398 INFO [StoreOpener-81254ceb5b41c7a9b3cb279f574da4d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:22,398 INFO [StoreOpener-81254ceb5b41c7a9b3cb279f574da4d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81254ceb5b41c7a9b3cb279f574da4d8 columnFamilyName family 2024-12-09T14:59:22,399 DEBUG [StoreOpener-81254ceb5b41c7a9b3cb279f574da4d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:59:22,399 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T14:59:22,399 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T14:59:22,399 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T14:59:22,399 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T14:59:22,400 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T14:59:22,401 DEBUG [StoreFileOpener-81254ceb5b41c7a9b3cb279f574da4d8-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/family/d9eacab60a2f45f39e3ad1151819319f, delay=996 ms 2024-12-09T14:59:22,401 DEBUG [RS:2;2491923345a5:41967 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T14:59:22,401 DEBUG [RS:2;2491923345a5:41967 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733756362397 2024-12-09T14:59:22,401 DEBUG [RS:2;2491923345a5:41967 {}] bucket.BucketCache(1509): Thread RS:2;2491923345a5:41967 finished persisting bucket cache to file, renaming 2024-12-09T14:59:22,402 INFO [RS:2;2491923345a5:41967 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T14:59:22,402 INFO [RS:2;2491923345a5:41967 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T14:59:22,402 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T14:59:22,402 INFO [RS:2;2491923345a5:41967 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T14:59:22,402 INFO [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer(959): stopping server 2491923345a5,41967,1733756361702 2024-12-09T14:59:22,402 INFO [RS:2;2491923345a5:41967 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:59:22,402 INFO [RS:2;2491923345a5:41967 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2491923345a5:41967. 2024-12-09T14:59:22,402 DEBUG [RS:2;2491923345a5:41967 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:59:22,402 DEBUG [RS:2;2491923345a5:41967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:22,402 DEBUG [StoreOpener-81254ceb5b41c7a9b3cb279f574da4d8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8/family/d9eacab60a2f45f39e3ad1151819319f 2024-12-09T14:59:22,402 INFO [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer(976): stopping server 2491923345a5,41967,1733756361702; all regions closed. 2024-12-09T14:59:22,402 INFO [regionserver/2491923345a5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:59:22,402 INFO [StoreOpener-81254ceb5b41c7a9b3cb279f574da4d8-1 {}] regionserver.HStore(327): Store=81254ceb5b41c7a9b3cb279f574da4d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:59:22,402 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:22,403 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:22,403 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/default/testBlockEvictionOnGracefulStop1733756359498/81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:22,404 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:22,404 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:22,404 DEBUG [RS:2;2491923345a5:41967 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs 2024-12-09T14:59:22,404 INFO [RS:2;2491923345a5:41967 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2491923345a5%2C41967%2C1733756361702:(num 1733756361937) 2024-12-09T14:59:22,404 DEBUG [RS:2;2491923345a5:41967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:22,404 INFO [RS:2;2491923345a5:41967 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:59:22,404 INFO [RS:2;2491923345a5:41967 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:59:22,404 INFO [RS:2;2491923345a5:41967 {}] hbase.ChoreService(370): Chore service for: regionserver/2491923345a5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T14:59:22,405 INFO [RS:2;2491923345a5:41967 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T14:59:22,405 INFO [RS:2;2491923345a5:41967 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T14:59:22,405 INFO [regionserver/2491923345a5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:59:22,405 INFO [RS:2;2491923345a5:41967 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T14:59:22,405 INFO [RS:2;2491923345a5:41967 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:59:22,405 INFO [RS:2;2491923345a5:41967 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41967 2024-12-09T14:59:22,405 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:22,406 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 81254ceb5b41c7a9b3cb279f574da4d8; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70393400, jitterRate=0.048943400382995605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:59:22,406 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81254ceb5b41c7a9b3cb279f574da4d8 2024-12-09T14:59:22,406 INFO [regionserver/2491923345a5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:59:22,406 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 81254ceb5b41c7a9b3cb279f574da4d8: Running coprocessor pre-open hook at 1733756362397Writing region info on filesystem at 1733756362397Initializing all the Stores at 1733756362397Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733756362397Cleaning up temporary data from old regions at 1733756362404 (+7 ms)Running coprocessor post-open hooks at 1733756362406 (+2 ms)Region opened successfully at 1733756362406 2024-12-09T14:59:22,406 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1722): Closing 81254ceb5b41c7a9b3cb279f574da4d8, disabling compactions & flushes 2024-12-09T14:59:22,406 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:22,407 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. 2024-12-09T14:59:22,407 WARN [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1802): Interrupted while waiting for close lock on testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:993) ~[?:?] at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.tryLock(ReentrantReadWriteLock.java:1131) ~[?:?] at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1794) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1672) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1627) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1610) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7817) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:59:22,407 DEBUG [RS:0;2491923345a5:40093 {}] bucket.BucketCache(1487): Thread RS:0;2491923345a5:40093 started persisting bucket cache to file 2024-12-09T14:59:22,407 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 81254ceb5b41c7a9b3cb279f574da4d8: Waiting for close lock at 1733756362406Running coprocessor pre-close hooks at 1733756362406Disabling compacts and flushes for region at 1733756362406 2024-12-09T14:59:22,407 DEBUG [RS:0;2491923345a5:40093 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T14:59:22,407 DEBUG [RS:0;2491923345a5:40093 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T14:59:22,407 WARN [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7819): Open region: {ENCODED => 81254ceb5b41c7a9b3cb279f574da4d8, NAME => 'testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8.', STARTKEY => '', ENDKEY => ''} failed. Try close region but got exception java.io.InterruptedIOException: Interrupted while waiting for close lock on testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1803) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1672) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1627) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1610) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7817) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:993) ~[?:?] at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.tryLock(ReentrantReadWriteLock.java:1131) ~[?:?] at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1794) ~[classes/:?] ... 13 more 2024-12-09T14:59:22,407 WARN [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(84): Failed to open region testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8., will report to master java.io.IOException: Cannot append; log is closed, regionName = testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1393) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.append(AbstractFSWAL.java:1920) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$appendMarker$14(AbstractFSWAL.java:1455) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendMarker(AbstractFSWAL.java:1455) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullMarkerAppendTransaction(WALUtil.java:168) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeMarker(WALUtil.java:148) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeRegionEventMarker(WALUtil.java:112) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.writeRegionOpenMarker(HRegion.java:1269) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7808) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:59:22,408 WARN [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(168): Fatal error occurred while opening region testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8., aborting... java.io.IOException: Failed to report failed open to master: testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.cleanUpAndReportFailure(AssignRegionHandler.java:93) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:145) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:59:22,409 ERROR [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2440): ***** ABORTING region server 2491923345a5,41967,1733756361702: Failed to open region testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. and can not recover ***** java.io.IOException: Failed to report failed open to master: testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8. at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.cleanUpAndReportFailure(AssignRegionHandler.java:93) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:145) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:59:22,409 ERROR [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T14:59:22,410 DEBUG [RS:0;2491923345a5:40093 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T14:59:22,410 DEBUG [RS:0;2491923345a5:40093 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733756362407 2024-12-09T14:59:22,410 DEBUG [RS:0;2491923345a5:40093 {}] bucket.BucketCache(1509): Thread RS:0;2491923345a5:40093 finished persisting bucket cache to file, renaming 2024-12-09T14:59:22,410 INFO [RS:0;2491923345a5:40093 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T14:59:22,411 INFO [RS:0;2491923345a5:40093 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T14:59:22,411 INFO [RS:0;2491923345a5:40093 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T14:59:22,411 INFO [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(959): stopping server 2491923345a5,40093,1733756358321 2024-12-09T14:59:22,411 INFO [RS:0;2491923345a5:40093 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:59:22,411 INFO [RS:0;2491923345a5:40093 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2491923345a5:40093. 2024-12-09T14:59:22,411 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T14:59:22,411 DEBUG [RS:0;2491923345a5:40093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:59:22,411 DEBUG [RS:0;2491923345a5:40093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:22,411 INFO [RS:0;2491923345a5:40093 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T14:59:22,411 INFO [RS:0;2491923345a5:40093 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T14:59:22,411 INFO [RS:0;2491923345a5:40093 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T14:59:22,411 INFO [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T14:59:22,412 INFO [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T14:59:22,412 DEBUG [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T14:59:22,412 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T14:59:22,412 DEBUG [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T14:59:22,412 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T14:59:22,412 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T14:59:22,412 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T14:59:22,412 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T14:59:22,412 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T14:59:22,412 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T14:59:22,412 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=485 B heapSize=1.70 KB 2024-12-09T14:59:22,415 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T14:59:22,416 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T14:59:22,422 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 385598832 }, "NonHeapMemoryUsage": { "committed": 98304000, "init": 7667712, "max": -1, "used": 95939840 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "2491923345a5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10693, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 44, "ProcessCallTime_min": 0, "ProcessCallTime_max": 12, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 6, "ProcessCallTime_75th_percentile": 9, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 44, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 44, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 44, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 44, "TotalCallTime_min": 0, "TotalCallTime_max": 12, "TotalCallTime_mean": 2, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 9, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 44, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 44, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 148, "ResponseSize_25th_percentile": 284, "ResponseSize_median": 569, "ResponseSize_75th_percentile": 854, "ResponseSize_90th_percentile": 1025, "ResponseSize_95th_percentile": 1082, "ResponseSize_98th_percentile": 1116, "ResponseSize_99th_percentile": 1127, "ResponseSize_99.9th_percentile": 1137, "ResponseSize_SizeRangeCount_0-10": 44, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 44, "RequestSize_min": 89, "RequestSize_max": 394, "RequestSize_mean": 174, "RequestSize_25th_percentile": 165, "RequestSize_median": 241, "RequestSize_75th_percentile": 317, "RequestSize_90th_percentile": 363, "RequestSize_95th_percentile": 378, "RequestSize_98th_percentile": 387, "RequestSize_99th_percentile": 390, "RequestSize_99.9th_percentile": 393, "RequestSize_SizeRangeCount_0-10": 44, "sentBytes": 5716 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "2491923345a5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:50273", "tag.serverName": "2491923345a5,43285,1733756352941", "tag.clusterId": "b0e674c5-788f-4076-baf5-8c61137e636e", "tag.Context": "regionserver", "tag.Hostname": "2491923345a5", "regionCount": 0, "storeCount": 0, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733756352941, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 1340218376, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 418148352, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 19931, "ByteBuffAllocatorPoolAllocationBytes": 399360, "ByteBuffAllocatorHeapAllocationRatio": 5.605830063265797E-4, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 21, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 7, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 13, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 7, "ScanTime_min": 0, "ScanTime_max": 3, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 2, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 7, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 13, "Put_min": 1, "Put_max": 7, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 13, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 23448, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 3867, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 2, "FlushOutputSize_min": 5038, "FlushOutputSize_max": 18410, "FlushOutputSize_mean": 11724, "FlushOutputSize_25th_percentile": 8381, "FlushOutputSize_median": 11724, "FlushOutputSize_75th_percentile": 15067, "FlushOutputSize_90th_percentile": 17072, "FlushOutputSize_95th_percentile": 17741, "FlushOutputSize_98th_percentile": 18142, "FlushOutputSize_99th_percentile": 18276, "FlushOutputSize_99.9th_percentile": 18396, "FlushOutputSize_SizeRangeCount_0-10": 2, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 2, "FlushMemstoreSize_min": 78, "FlushMemstoreSize_max": 3789, "FlushMemstoreSize_mean": 1933, "FlushMemstoreSize_25th_percentile": 1005, "FlushMemstoreSize_median": 1933, "FlushMemstoreSize_75th_percentile": 2861, "FlushMemstoreSize_90th_percentile": 3417, "FlushMemstoreSize_95th_percentile": 3603, "FlushMemstoreSize_98th_percentile": 3714, "FlushMemstoreSize_99th_percentile": 3751, "FlushMemstoreSize_99.9th_percentile": 3785, "FlushMemstoreSize_SizeRangeCount_0-10": 2, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 2, "FlushTime_min": 81, "FlushTime_max": 98, "FlushTime_mean": 89, "FlushTime_25th_percentile": 85, "FlushTime_median": 89, "FlushTime_75th_percentile": 93, "FlushTime_90th_percentile": 96, "FlushTime_95th_percentile": 97, "FlushTime_98th_percentile": 97, "FlushTime_99th_percentile": 97, "FlushTime_99.9th_percentile": 97, "FlushTime_TimeRangeCount_0-1": 2, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 7, "ScanSize_min": 0, "ScanSize_max": 1112, "ScanSize_mean": 517, "ScanSize_25th_percentile": 278, "ScanSize_median": 556, "ScanSize_75th_percentile": 834, "ScanSize_90th_percentile": 1000, "ScanSize_95th_percentile": 1056, "ScanSize_98th_percentile": 1089, "ScanSize_99th_percentile": 1100, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_0-10": 7, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T14:59:22,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:59:22,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41967-0x1000b244c8f0003, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2491923345a5,41967,1733756361702 2024-12-09T14:59:22,424 INFO [RS:2;2491923345a5:41967 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:59:22,424 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2491923345a5,41967,1733756361702] 2024-12-09T14:59:22,430 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0cdf37f0ae714aa4b5ca55f185b7d9a0 2024-12-09T14:59:22,442 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/info/0cdf37f0ae714aa4b5ca55f185b7d9a0 is 209, key is testBlockEvictionOnGracefulStop1733756359498,,1733756359502.81254ceb5b41c7a9b3cb279f574da4d8./info:regioninfo/1733756362116/Put/seqid=0 2024-12-09T14:59:22,442 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=485 B at sequenceid=18 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/info/0cdf37f0ae714aa4b5ca55f185b7d9a0 2024-12-09T14:59:22,445 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/.tmp/info/0cdf37f0ae714aa4b5ca55f185b7d9a0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/0cdf37f0ae714aa4b5ca55f185b7d9a0 2024-12-09T14:59:22,446 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/0cdf37f0ae714aa4b5ca55f185b7d9a0, delay=937 ms 2024-12-09T14:59:22,447 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/0cdf37f0ae714aa4b5ca55f185b7d9a0, entries=3, sequenceid=18, filesize=5.9 K 2024-12-09T14:59:22,447 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=18, compaction requested=false 2024-12-09T14:59:22,448 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/f37dbe1cee74437693532b99bdf57efa 2024-12-09T14:59:22,448 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/info/0cdf37f0ae714aa4b5ca55f185b7d9a0 2024-12-09T14:59:22,449 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/ns/d40f7c95b1a6492c91342f6dee25dc74 2024-12-09T14:59:22,449 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/table/cb60e9ebf9cf46149d6a24756082cde3 2024-12-09T14:59:22,450 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2491923345a5,41967,1733756361702 already deleted, retry=false 2024-12-09T14:59:22,450 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2491923345a5,41967,1733756361702 expired; onlineServers=1 2024-12-09T14:59:22,453 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/data/hbase/meta/1588230740/recovered.edits/21.seqid, newMaxSeqId=21, maxSeqId=14 2024-12-09T14:59:22,453 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T14:59:22,453 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T14:59:22,453 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733756362412Running coprocessor pre-close hooks at 1733756362412Disabling compacts and flushes for region at 1733756362412Disabling writes for close at 1733756362412Obtaining lock to block concurrent updates at 1733756362412Preparing flush snapshotting stores in 1588230740 at 1733756362412Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=485, getHeapSize=1672, getOffHeapSize=0, getCellsCount=3 at 1733756362413 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733756362413Flushing 1588230740/info: creating writer at 1733756362413Flushing 1588230740/info: appending metadata at 1733756362441 (+28 ms)Flushing 1588230740/info: closing flushed file at 1733756362441Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4433b858: reopening flushed file at 1733756362445 (+4 ms)Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=18, compaction requested=false at 1733756362447 (+2 ms)Writing region close event to WAL at 1733756362450 (+3 ms)Running coprocessor post-close hooks at 1733756362453 (+3 ms)Closed at 1733756362453 2024-12-09T14:59:22,454 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T14:59:22,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41967-0x1000b244c8f0003, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:22,543 INFO [RS:2;2491923345a5:41967 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:59:22,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41967-0x1000b244c8f0003, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:22,543 INFO [RS:2;2491923345a5:41967 {}] regionserver.HRegionServer(1031): Exiting; stopping=2491923345a5,41967,1733756361702; zookeeper connection closed. 2024-12-09T14:59:22,544 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5db3f517 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5db3f517 2024-12-09T14:59:22,612 INFO [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(976): stopping server 2491923345a5,40093,1733756358321; all regions closed. 2024-12-09T14:59:22,617 DEBUG [RS:0;2491923345a5:40093 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs 2024-12-09T14:59:22,618 INFO [RS:0;2491923345a5:40093 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2491923345a5%2C40093%2C1733756358321.meta:.meta(num 1733756361884) 2024-12-09T14:59:22,622 DEBUG [RS:0;2491923345a5:40093 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/oldWALs 2024-12-09T14:59:22,622 INFO [RS:0;2491923345a5:40093 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2491923345a5%2C40093%2C1733756358321:(num 1733756359032) 2024-12-09T14:59:22,623 DEBUG [RS:0;2491923345a5:40093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:59:22,623 INFO [RS:0;2491923345a5:40093 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:59:22,623 INFO [RS:0;2491923345a5:40093 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:59:22,623 INFO [RS:0;2491923345a5:40093 {}] hbase.ChoreService(370): Chore service for: regionserver/2491923345a5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T14:59:22,623 INFO [RS:0;2491923345a5:40093 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:59:22,623 INFO [regionserver/2491923345a5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:59:22,623 INFO [RS:0;2491923345a5:40093 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40093 2024-12-09T14:59:22,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:59:22,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2491923345a5,40093,1733756358321 2024-12-09T14:59:22,659 INFO [RS:0;2491923345a5:40093 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:59:22,671 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2491923345a5,40093,1733756358321] 2024-12-09T14:59:22,679 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2491923345a5,40093,1733756358321 already deleted, retry=false 2024-12-09T14:59:22,679 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2491923345a5,40093,1733756358321 expired; onlineServers=0 2024-12-09T14:59:22,679 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2491923345a5,39975,1733756358144' ***** 2024-12-09T14:59:22,679 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T14:59:22,679 INFO [M:0;2491923345a5:39975 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:59:22,680 INFO [M:0;2491923345a5:39975 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:59:22,680 DEBUG [M:0;2491923345a5:39975 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T14:59:22,680 DEBUG [M:0;2491923345a5:39975 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T14:59:22,680 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T14:59:22,680 DEBUG [master/2491923345a5:0:becomeActiveMaster-HFileCleaner.small.0-1733756358723 {}] cleaner.HFileCleaner(306): Exit Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.small.0-1733756358723,5,FailOnTimeoutGroup] 2024-12-09T14:59:22,680 DEBUG [master/2491923345a5:0:becomeActiveMaster-HFileCleaner.large.0-1733756358723 {}] cleaner.HFileCleaner(306): Exit Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.large.0-1733756358723,5,FailOnTimeoutGroup] 2024-12-09T14:59:22,680 INFO [M:0;2491923345a5:39975 {}] hbase.ChoreService(370): Chore service for: master/2491923345a5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T14:59:22,680 INFO [M:0;2491923345a5:39975 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:59:22,680 DEBUG [M:0;2491923345a5:39975 {}] master.HMaster(1795): Stopping service threads 2024-12-09T14:59:22,681 INFO [M:0;2491923345a5:39975 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T14:59:22,681 INFO [M:0;2491923345a5:39975 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T14:59:22,681 INFO [M:0;2491923345a5:39975 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T14:59:22,681 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T14:59:22,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T14:59:22,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:59:22,688 DEBUG [M:0;2491923345a5:39975 {}] zookeeper.ZKUtil(347): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T14:59:22,688 WARN [M:0;2491923345a5:39975 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T14:59:22,688 INFO [M:0;2491923345a5:39975 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/.lastflushedseqids 2024-12-09T14:59:22,695 INFO [M:0;2491923345a5:39975 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T14:59:22,695 INFO [M:0;2491923345a5:39975 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T14:59:22,695 DEBUG [M:0;2491923345a5:39975 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T14:59:22,695 INFO [M:0;2491923345a5:39975 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:22,696 DEBUG [M:0;2491923345a5:39975 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:22,696 DEBUG [M:0;2491923345a5:39975 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T14:59:22,696 DEBUG [M:0;2491923345a5:39975 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:22,696 INFO [M:0;2491923345a5:39975 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.95 KB heapSize=44.17 KB 2024-12-09T14:59:22,704 DEBUG [M:0;2491923345a5:39975 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 11cae327afcb4617a23cdd89e8445e6c 2024-12-09T14:59:22,715 DEBUG [M:0;2491923345a5:39975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/11cae327afcb4617a23cdd89e8445e6c is 82, key is hbase:meta,,1/info:regioninfo/1733756361922/Put/seqid=0 2024-12-09T14:59:22,716 INFO [M:0;2491923345a5:39975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/11cae327afcb4617a23cdd89e8445e6c 2024-12-09T14:59:22,722 DEBUG [M:0;2491923345a5:39975 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fc5043a8ed82432e8da414f56fc7c5f3 2024-12-09T14:59:22,735 DEBUG [M:0;2491923345a5:39975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc5043a8ed82432e8da414f56fc7c5f3 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733756359880/Put/seqid=0 2024-12-09T14:59:22,735 INFO [M:0;2491923345a5:39975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.73 KB at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc5043a8ed82432e8da414f56fc7c5f3 2024-12-09T14:59:22,741 DEBUG [M:0;2491923345a5:39975 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on eedc946b42dd4cd494a8fe6f8452191e 2024-12-09T14:59:22,754 DEBUG [M:0;2491923345a5:39975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eedc946b42dd4cd494a8fe6f8452191e is 69, key is 2491923345a5,40093,1733756358321/rs:state/1733756358846/Put/seqid=0 2024-12-09T14:59:22,755 INFO [M:0;2491923345a5:39975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eedc946b42dd4cd494a8fe6f8452191e 2024-12-09T14:59:22,756 INFO [M:0;2491923345a5:39975 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eedc946b42dd4cd494a8fe6f8452191e 2024-12-09T14:59:22,757 DEBUG [M:0;2491923345a5:39975 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/11cae327afcb4617a23cdd89e8445e6c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/11cae327afcb4617a23cdd89e8445e6c 2024-12-09T14:59:22,758 INFO [M:0;2491923345a5:39975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/11cae327afcb4617a23cdd89e8445e6c, entries=14, sequenceid=90, filesize=5.9 K 2024-12-09T14:59:22,759 DEBUG [M:0;2491923345a5:39975 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc5043a8ed82432e8da414f56fc7c5f3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fc5043a8ed82432e8da414f56fc7c5f3 2024-12-09T14:59:22,761 INFO [M:0;2491923345a5:39975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fc5043a8ed82432e8da414f56fc7c5f3, entries=11, sequenceid=90, filesize=7.8 K 2024-12-09T14:59:22,761 DEBUG [M:0;2491923345a5:39975 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eedc946b42dd4cd494a8fe6f8452191e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eedc946b42dd4cd494a8fe6f8452191e 2024-12-09T14:59:22,763 INFO [M:0;2491923345a5:39975 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eedc946b42dd4cd494a8fe6f8452191e 2024-12-09T14:59:22,763 INFO [M:0;2491923345a5:39975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/9215b300-345f-0c4f-886e-6e3a564d952b/9ce90a34-e9e4-0186-0b21-fdd2e799a47a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eedc946b42dd4cd494a8fe6f8452191e, entries=3, sequenceid=90, filesize=5.3 K 2024-12-09T14:59:22,764 INFO [M:0;2491923345a5:39975 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.95 KB/35792, heapSize ~43.88 KB/44928, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=90, compaction requested=false 2024-12-09T14:59:22,766 INFO [M:0;2491923345a5:39975 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:59:22,766 DEBUG [M:0;2491923345a5:39975 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733756362695Disabling compacts and flushes for region at 1733756362695Disabling writes for close at 1733756362696 (+1 ms)Obtaining lock to block concurrent updates at 1733756362696Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733756362696Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35792, getHeapSize=45168, getOffHeapSize=0, getCellsCount=111 at 1733756362696Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733756362696Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733756362697 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733756362715 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733756362715Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733756362717 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733756362734 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733756362734Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733756362737 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733756362753 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733756362753Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74b84f1e: reopening flushed file at 1733756362756 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5430391a: reopening flushed file at 1733756362759 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ce8c0b4: reopening flushed file at 1733756362761 (+2 ms)Finished flush of dataSize ~34.95 KB/35792, heapSize ~43.88 KB/44928, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=90, compaction requested=false at 1733756362764 (+3 ms)Writing region close event to WAL at 1733756362766 (+2 ms)Closed at 1733756362766 2024-12-09T14:59:22,767 INFO [M:0;2491923345a5:39975 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T14:59:22,767 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:59:22,767 INFO [M:0;2491923345a5:39975 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39975 2024-12-09T14:59:22,768 INFO [M:0;2491923345a5:39975 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:59:22,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:22,771 INFO [RS:0;2491923345a5:40093 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:59:22,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40093-0x1000b244c8f0001, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:22,771 INFO [RS:0;2491923345a5:40093 {}] regionserver.HRegionServer(1031): Exiting; stopping=2491923345a5,40093,1733756358321; zookeeper connection closed. 2024-12-09T14:59:22,772 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@11dbad1b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@11dbad1b 2024-12-09T14:59:22,772 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-09T14:59:22,884 INFO [M:0;2491923345a5:39975 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:59:22,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:22,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39975-0x1000b244c8f0000, quorum=127.0.0.1:56049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:59:22,886 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T14:59:22,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-09T14:59:22,887 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:59:22,894 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=78 (was 49) Potentially hanging thread: NIOWorkerThread-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:938) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:994) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-14 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:938) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:994) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:938) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:994) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:938) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:994) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:938) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:994) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-16 java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:1005) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:938) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) java.base@17.0.11/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:994) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=383 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=316 (was 308) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7691 (was 7789)