2024-12-05 00:44:27,130 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-05 00:44:27,148 main DEBUG Took 0.016148 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 00:44:27,148 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 00:44:27,149 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 00:44:27,149 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 00:44:27,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,158 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 00:44:27,172 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,174 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,174 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,175 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,177 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,177 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,178 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,179 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,179 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,180 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,180 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,181 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,181 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,182 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,182 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,182 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,183 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,183 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,184 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,184 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,185 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:44:27,185 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,186 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 00:44:27,187 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:44:27,189 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 00:44:27,191 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 00:44:27,192 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 00:44:27,193 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 00:44:27,194 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 00:44:27,203 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 00:44:27,206 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 00:44:27,208 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 00:44:27,208 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 00:44:27,209 main DEBUG createAppenders(={Console}) 2024-12-05 00:44:27,209 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-05 00:44:27,210 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-05 00:44:27,210 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-05 00:44:27,211 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 00:44:27,211 main DEBUG OutputStream closed 2024-12-05 00:44:27,211 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 00:44:27,212 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 00:44:27,212 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-05 00:44:27,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 00:44:27,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 00:44:27,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 00:44:27,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 00:44:27,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 00:44:27,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 00:44:27,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 00:44:27,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 00:44:27,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 00:44:27,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 00:44:27,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 00:44:27,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 00:44:27,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 00:44:27,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 00:44:27,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 00:44:27,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 00:44:27,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 00:44:27,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 00:44:27,295 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 00:44:27,296 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-05 00:44:27,296 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 00:44:27,296 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-05T00:44:27,307 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-05 00:44:27,309 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 00:44:27,309 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T00:44:27,513 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd 2024-12-05T00:44:27,537 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=294, ProcessCount=11, AvailableMemoryMB=10562 2024-12-05T00:44:27,622 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T00:44:27,707 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:27,722 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/cluster_76daac72-d75e-eb0f-7b5c-b4d4c16bdd9c, deleteOnExit=true 2024-12-05T00:44:27,818 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/cluster_76daac72-d75e-eb0f-7b5c-b4d4c16bdd9c/zookeeper_0, clientPort=57216, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/cluster_76daac72-d75e-eb0f-7b5c-b4d4c16bdd9c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/cluster_76daac72-d75e-eb0f-7b5c-b4d4c16bdd9c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T00:44:27,830 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57216 2024-12-05T00:44:27,830 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:27,899 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b with version=8 2024-12-05T00:44:27,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/hbase-staging 2024-12-05T00:44:27,979 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T00:44:28,199 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fea72ea5c4b6:0 server-side Connection retries=45 2024-12-05T00:44:28,207 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:28,208 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:28,215 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T00:44:28,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:28,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T00:44:28,325 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T00:44:28,407 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T00:44:28,449 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T00:44:28,458 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T00:44:28,462 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T00:44:28,496 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 11430 (auto-detected) 2024-12-05T00:44:28,497 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T00:44:28,517 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35373 2024-12-05T00:44:28,536 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35373 connecting to ZooKeeper ensemble=127.0.0.1:57216 2024-12-05T00:44:28,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:353730x0, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T00:44:28,649 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35373-0x101a2f278c30000 connected 2024-12-05T00:44:28,760 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:28,761 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:28,771 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:28,775 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b, hbase.cluster.distributed=false 2024-12-05T00:44:28,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T00:44:28,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35373 2024-12-05T00:44:28,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35373 2024-12-05T00:44:28,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35373 2024-12-05T00:44:28,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35373 2024-12-05T00:44:28,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35373 2024-12-05T00:44:28,874 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fea72ea5c4b6:0 server-side Connection retries=45 2024-12-05T00:44:28,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:28,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:28,876 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T00:44:28,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:28,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T00:44:28,879 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T00:44:28,881 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T00:44:28,882 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37427 2024-12-05T00:44:28,884 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37427 connecting to ZooKeeper ensemble=127.0.0.1:57216 2024-12-05T00:44:28,884 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:28,885 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:28,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374270x0, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T00:44:28,899 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:374270x0, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:28,899 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37427-0x101a2f278c30001 connected 2024-12-05T00:44:28,906 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T00:44:28,918 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/bucket.cache 2024-12-05T00:44:28,922 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T00:44:28,923 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T00:44:28,925 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T00:44:28,927 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T00:44:28,927 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T00:44:28,931 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T00:44:28,937 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T00:44:28,937 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T00:44:28,938 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/bucket.persistence is complete. 2024-12-05T00:44:28,939 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T00:44:28,944 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T00:44:28,945 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37427 2024-12-05T00:44:28,946 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37427 2024-12-05T00:44:28,946 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37427 2024-12-05T00:44:28,947 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37427 2024-12-05T00:44:28,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37427 2024-12-05T00:44:28,969 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fea72ea5c4b6:0 server-side Connection retries=45 2024-12-05T00:44:28,969 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:28,970 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:28,970 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T00:44:28,970 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:28,970 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T00:44:28,971 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T00:44:28,971 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T00:44:28,972 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34765 2024-12-05T00:44:28,974 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34765 connecting to ZooKeeper ensemble=127.0.0.1:57216 2024-12-05T00:44:28,975 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:28,975 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:28,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347650x0, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T00:44:28,987 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:347650x0, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:28,987 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34765-0x101a2f278c30002 connected 2024-12-05T00:44:28,988 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T00:44:28,989 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/bucket.cache 2024-12-05T00:44:28,989 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T00:44:28,990 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T00:44:28,990 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T00:44:28,990 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T00:44:28,990 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T00:44:28,991 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T00:44:28,991 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T00:44:28,991 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/bucket.persistence is complete. 2024-12-05T00:44:28,992 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T00:44:28,994 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T00:44:28,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T00:44:28,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34765 2024-12-05T00:44:28,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34765 2024-12-05T00:44:29,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34765 2024-12-05T00:44:29,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34765 2024-12-05T00:44:29,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34765 2024-12-05T00:44:29,024 DEBUG [M:0;fea72ea5c4b6:35373 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fea72ea5c4b6:35373 2024-12-05T00:44:29,025 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fea72ea5c4b6,35373,1733359468043 2024-12-05T00:44:29,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:29,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:29,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:29,043 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fea72ea5c4b6,35373,1733359468043 2024-12-05T00:44:29,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T00:44:29,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T00:44:29,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,082 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T00:44:29,083 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fea72ea5c4b6,35373,1733359468043 from backup master directory 2024-12-05T00:44:29,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:29,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:29,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fea72ea5c4b6,35373,1733359468043 2024-12-05T00:44:29,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:29,092 WARN [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T00:44:29,092 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fea72ea5c4b6,35373,1733359468043 2024-12-05T00:44:29,094 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T00:44:29,096 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T00:44:29,102 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/hbase.id] with ID: 038e0f87-a58c-43fb-a8a9-e9242d89242c 2024-12-05T00:44:29,102 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/.tmp/hbase.id 2024-12-05T00:44:29,107 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/hbase.id] 2024-12-05T00:44:29,143 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:29,150 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T00:44:29,171 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-05T00:44:29,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,215 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:44:29,219 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T00:44:29,257 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:44:29,262 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:44:29,310 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store 2024-12-05T00:44:29,330 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T00:44:29,333 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:29,334 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T00:44:29,334 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:29,335 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:29,336 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T00:44:29,336 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:29,336 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:29,337 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733359469334Disabling compacts and flushes for region at 1733359469334Disabling writes for close at 1733359469336 (+2 ms)Writing region close event to WAL at 1733359469336Closed at 1733359469336 2024-12-05T00:44:29,337 WARN [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/.initializing 2024-12-05T00:44:29,338 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/WALs/fea72ea5c4b6,35373,1733359468043 2024-12-05T00:44:29,352 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T00:44:29,366 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fea72ea5c4b6%2C35373%2C1733359468043, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/WALs/fea72ea5c4b6,35373,1733359468043, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/oldWALs, maxLogs=10 2024-12-05T00:44:29,402 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/WALs/fea72ea5c4b6,35373,1733359468043/fea72ea5c4b6%2C35373%2C1733359468043.1733359469371 2024-12-05T00:44:29,403 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T00:44:29,404 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:44:29,405 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:29,410 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,411 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,458 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,498 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T00:44:29,504 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:29,507 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:29,508 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,513 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T00:44:29,514 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:29,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:44:29,515 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,520 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T00:44:29,520 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:29,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:44:29,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T00:44:29,526 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:29,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:44:29,527 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,531 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,532 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,535 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,536 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,539 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T00:44:29,541 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:29,548 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:44:29,549 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61935461, jitterRate=-0.07708971202373505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T00:44:29,557 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733359469424Initializing all the Stores at 1733359469424Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359469425 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359469425Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359469426 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359469426Cleaning up temporary data from old regions at 1733359469536 (+110 ms)Region opened successfully at 1733359469557 (+21 ms) 2024-12-05T00:44:29,559 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T00:44:29,589 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f9dad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fea72ea5c4b6/172.17.0.2:0 2024-12-05T00:44:29,624 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T00:44:29,635 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T00:44:29,636 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T00:44:29,639 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T00:44:29,640 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T00:44:29,646 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-05T00:44:29,646 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T00:44:29,672 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T00:44:29,680 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T00:44:29,730 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T00:44:29,732 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T00:44:29,734 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T00:44:29,743 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T00:44:29,745 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T00:44:29,749 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T00:44:29,761 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T00:44:29,764 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T00:44:29,775 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T00:44:29,795 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T00:44:29,806 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T00:44:29,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:29,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:29,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:29,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,831 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fea72ea5c4b6,35373,1733359468043, sessionid=0x101a2f278c30000, setting cluster-up flag (Was=false) 2024-12-05T00:44:29,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,891 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T00:44:29,895 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fea72ea5c4b6,35373,1733359468043 2024-12-05T00:44:29,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:29,954 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T00:44:29,958 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fea72ea5c4b6,35373,1733359468043 2024-12-05T00:44:29,967 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T00:44:30,013 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer(746): ClusterId : 038e0f87-a58c-43fb-a8a9-e9242d89242c 2024-12-05T00:44:30,013 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(746): ClusterId : 038e0f87-a58c-43fb-a8a9-e9242d89242c 2024-12-05T00:44:30,015 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T00:44:30,015 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T00:44:30,030 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T00:44:30,030 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T00:44:30,030 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T00:44:30,030 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T00:44:30,037 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T00:44:30,040 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T00:44:30,040 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T00:44:30,041 DEBUG [RS:1;fea72ea5c4b6:34765 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20aea261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fea72ea5c4b6/172.17.0.2:0 2024-12-05T00:44:30,041 DEBUG [RS:0;fea72ea5c4b6:37427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c36af02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fea72ea5c4b6/172.17.0.2:0 2024-12-05T00:44:30,048 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T00:44:30,056 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T00:44:30,060 DEBUG [RS:1;fea72ea5c4b6:34765 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fea72ea5c4b6:34765 2024-12-05T00:44:30,060 DEBUG [RS:0;fea72ea5c4b6:37427 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fea72ea5c4b6:37427 2024-12-05T00:44:30,064 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T00:44:30,064 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T00:44:30,064 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T00:44:30,064 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T00:44:30,064 DEBUG [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T00:44:30,064 DEBUG [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T00:44:30,061 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fea72ea5c4b6,35373,1733359468043 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T00:44:30,066 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer(2659): reportForDuty to master=fea72ea5c4b6,35373,1733359468043 with port=34765, startcode=1733359468969 2024-12-05T00:44:30,066 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(2659): reportForDuty to master=fea72ea5c4b6,35373,1733359468043 with port=37427, startcode=1733359468844 2024-12-05T00:44:30,070 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:44:30,070 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:44:30,070 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:44:30,070 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:44:30,070 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fea72ea5c4b6:0, corePoolSize=10, maxPoolSize=10 2024-12-05T00:44:30,071 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,071 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=2, maxPoolSize=2 2024-12-05T00:44:30,071 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,075 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T00:44:30,075 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T00:44:30,077 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733359500077 2024-12-05T00:44:30,078 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T00:44:30,078 DEBUG [RS:1;fea72ea5c4b6:34765 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T00:44:30,078 DEBUG [RS:0;fea72ea5c4b6:37427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T00:44:30,079 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T00:44:30,080 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:30,080 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T00:44:30,082 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T00:44:30,083 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T00:44:30,083 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T00:44:30,083 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T00:44:30,084 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,087 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T00:44:30,088 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T00:44:30,089 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T00:44:30,090 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T00:44:30,090 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T00:44:30,092 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.large.0-1733359470091,5,FailOnTimeoutGroup] 2024-12-05T00:44:30,093 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.small.0-1733359470092,5,FailOnTimeoutGroup] 2024-12-05T00:44:30,093 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,093 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T00:44:30,094 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,095 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,101 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T00:44:30,101 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b 2024-12-05T00:44:30,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:30,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T00:44:30,115 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T00:44:30,115 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:30,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:30,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T00:44:30,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T00:44:30,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:30,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:30,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T00:44:30,133 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T00:44:30,133 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:30,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:30,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T00:44:30,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T00:44:30,139 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:30,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:30,139 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T00:44:30,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740 2024-12-05T00:44:30,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740 2024-12-05T00:44:30,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T00:44:30,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T00:44:30,142 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T00:44:30,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T00:44:30,145 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59111, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T00:44:30,145 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57677, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T00:44:30,150 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:44:30,151 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69134926, jitterRate=0.03019067645072937}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T00:44:30,152 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35373 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:30,154 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35373 {}] master.ServerManager(517): Registering regionserver=fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:30,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733359470109Initializing all the Stores at 1733359470109Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359470109Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359470110 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359470110Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359470110Cleaning up temporary data from old regions at 1733359470141 (+31 ms)Region opened successfully at 1733359470154 (+13 ms) 2024-12-05T00:44:30,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T00:44:30,155 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T00:44:30,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T00:44:30,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T00:44:30,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T00:44:30,157 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T00:44:30,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733359470155Disabling compacts and flushes for region at 1733359470155Disabling writes for close at 1733359470155Writing region close event to WAL at 1733359470157 (+2 ms)Closed at 1733359470157 2024-12-05T00:44:30,160 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T00:44:30,161 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T00:44:30,164 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35373 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fea72ea5c4b6,34765,1733359468969 2024-12-05T00:44:30,164 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35373 {}] master.ServerManager(517): Registering regionserver=fea72ea5c4b6,34765,1733359468969 2024-12-05T00:44:30,167 DEBUG [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b 2024-12-05T00:44:30,167 DEBUG [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b 2024-12-05T00:44:30,168 DEBUG [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T00:44:30,168 DEBUG [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T00:44:30,168 DEBUG [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T00:44:30,168 DEBUG [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T00:44:30,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T00:44:30,176 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T00:44:30,178 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T00:44:30,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T00:44:30,225 DEBUG [RS:0;fea72ea5c4b6:37427 {}] zookeeper.ZKUtil(111): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:30,225 DEBUG [RS:1;fea72ea5c4b6:34765 {}] zookeeper.ZKUtil(111): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fea72ea5c4b6,34765,1733359468969 2024-12-05T00:44:30,225 WARN [RS:0;fea72ea5c4b6:37427 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T00:44:30,225 WARN [RS:1;fea72ea5c4b6:34765 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T00:44:30,225 INFO [RS:0;fea72ea5c4b6:37427 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:44:30,225 INFO [RS:1;fea72ea5c4b6:34765 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:44:30,225 DEBUG [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:30,225 DEBUG [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,34765,1733359468969 2024-12-05T00:44:30,228 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fea72ea5c4b6,34765,1733359468969] 2024-12-05T00:44:30,228 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fea72ea5c4b6,37427,1733359468844] 2024-12-05T00:44:30,255 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T00:44:30,255 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T00:44:30,269 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T00:44:30,269 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T00:44:30,273 INFO [RS:0;fea72ea5c4b6:37427 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T00:44:30,273 INFO [RS:1;fea72ea5c4b6:34765 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T00:44:30,273 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,273 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,274 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T00:44:30,276 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T00:44:30,280 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T00:44:30,280 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T00:44:30,282 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,282 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,282 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,282 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,283 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,283 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,283 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,283 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,283 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,283 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,283 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,283 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,283 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=2, maxPoolSize=2 2024-12-05T00:44:30,283 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=2, maxPoolSize=2 2024-12-05T00:44:30,283 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,283 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,284 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,284 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,284 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,284 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,284 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,284 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,284 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,284 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,284 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,284 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:30,284 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:44:30,285 DEBUG [RS:0;fea72ea5c4b6:37427 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:44:30,285 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:44:30,285 DEBUG [RS:1;fea72ea5c4b6:34765 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:44:30,288 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,288 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,289 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,289 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,289 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,289 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,289 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,289 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,289 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,289 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,289 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,34765,1733359468969-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T00:44:30,289 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,37427,1733359468844-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T00:44:30,309 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T00:44:30,309 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T00:44:30,311 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,34765,1733359468969-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,311 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,37427,1733359468844-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,312 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,312 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,312 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.Replication(171): fea72ea5c4b6,37427,1733359468844 started 2024-12-05T00:44:30,312 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.Replication(171): fea72ea5c4b6,34765,1733359468969 started 2024-12-05T00:44:30,329 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,329 WARN [fea72ea5c4b6:35373 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T00:44:30,330 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(1482): Serving as fea72ea5c4b6,37427,1733359468844, RpcServer on fea72ea5c4b6/172.17.0.2:37427, sessionid=0x101a2f278c30001 2024-12-05T00:44:30,331 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T00:44:30,331 DEBUG [RS:0;fea72ea5c4b6:37427 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:30,331 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,37427,1733359468844' 2024-12-05T00:44:30,332 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T00:44:30,333 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T00:44:30,334 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T00:44:30,334 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T00:44:30,334 DEBUG [RS:0;fea72ea5c4b6:37427 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:30,334 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,37427,1733359468844' 2024-12-05T00:44:30,334 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T00:44:30,335 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T00:44:30,336 DEBUG [RS:0;fea72ea5c4b6:37427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T00:44:30,336 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:30,336 INFO [RS:0;fea72ea5c4b6:37427 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T00:44:30,336 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer(1482): Serving as fea72ea5c4b6,34765,1733359468969, RpcServer on fea72ea5c4b6/172.17.0.2:34765, sessionid=0x101a2f278c30002 2024-12-05T00:44:30,336 INFO [RS:0;fea72ea5c4b6:37427 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T00:44:30,336 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T00:44:30,336 DEBUG [RS:1;fea72ea5c4b6:34765 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fea72ea5c4b6,34765,1733359468969 2024-12-05T00:44:30,337 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,34765,1733359468969' 2024-12-05T00:44:30,337 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T00:44:30,337 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T00:44:30,338 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T00:44:30,338 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T00:44:30,338 DEBUG [RS:1;fea72ea5c4b6:34765 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fea72ea5c4b6,34765,1733359468969 2024-12-05T00:44:30,338 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,34765,1733359468969' 2024-12-05T00:44:30,339 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T00:44:30,339 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T00:44:30,340 DEBUG [RS:1;fea72ea5c4b6:34765 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T00:44:30,340 INFO [RS:1;fea72ea5c4b6:34765 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T00:44:30,340 INFO [RS:1;fea72ea5c4b6:34765 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T00:44:30,442 INFO [RS:1;fea72ea5c4b6:34765 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T00:44:30,442 INFO [RS:0;fea72ea5c4b6:37427 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T00:44:30,443 INFO [RS:1;fea72ea5c4b6:34765 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fea72ea5c4b6%2C34765%2C1733359468969, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,34765,1733359468969, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs, maxLogs=55 2024-12-05T00:44:30,443 INFO [RS:0;fea72ea5c4b6:37427 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fea72ea5c4b6%2C37427%2C1733359468844, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,37427,1733359468844, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs, maxLogs=55 2024-12-05T00:44:30,462 INFO [RS:1;fea72ea5c4b6:34765 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,34765,1733359468969/fea72ea5c4b6%2C34765%2C1733359468969.1733359470446 2024-12-05T00:44:30,463 DEBUG [RS:1;fea72ea5c4b6:34765 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T00:44:30,469 INFO [RS:0;fea72ea5c4b6:37427 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,37427,1733359468844/fea72ea5c4b6%2C37427%2C1733359468844.1733359470446 2024-12-05T00:44:30,471 DEBUG [RS:0;fea72ea5c4b6:37427 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T00:44:30,585 DEBUG [fea72ea5c4b6:35373 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-05T00:44:30,595 DEBUG [fea72ea5c4b6:35373 {}] balancer.BalancerClusterState(204): Hosts are {fea72ea5c4b6=0} racks are {/default-rack=0} 2024-12-05T00:44:30,597 DEBUG [fea72ea5c4b6:35373 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T00:44:30,597 DEBUG [fea72ea5c4b6:35373 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T00:44:30,597 DEBUG [fea72ea5c4b6:35373 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T00:44:30,598 DEBUG [fea72ea5c4b6:35373 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T00:44:30,598 INFO [fea72ea5c4b6:35373 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T00:44:30,598 INFO [fea72ea5c4b6:35373 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T00:44:30,598 DEBUG [fea72ea5c4b6:35373 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T00:44:30,604 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:30,608 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fea72ea5c4b6,37427,1733359468844, state=OPENING 2024-12-05T00:44:30,709 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T00:44:30,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:30,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:30,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:30,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:30,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:30,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:30,722 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T00:44:30,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fea72ea5c4b6,37427,1733359468844}] 2024-12-05T00:44:30,893 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T00:44:30,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39113, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T00:44:30,902 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T00:44:30,903 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:44:30,903 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T00:44:30,905 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fea72ea5c4b6%2C37427%2C1733359468844.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,37427,1733359468844, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs, maxLogs=55 2024-12-05T00:44:30,923 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,37427,1733359468844/fea72ea5c4b6%2C37427%2C1733359468844.meta.1733359470907.meta 2024-12-05T00:44:30,924 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T00:44:30,924 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:44:30,926 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T00:44:30,928 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T00:44:30,932 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T00:44:30,935 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T00:44:30,936 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:30,936 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T00:44:30,936 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T00:44:30,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T00:44:30,939 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T00:44:30,939 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:30,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:30,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T00:44:30,940 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T00:44:30,940 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:30,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:30,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T00:44:30,941 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T00:44:30,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:30,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:30,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T00:44:30,943 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T00:44:30,943 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:30,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:30,943 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T00:44:30,944 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740 2024-12-05T00:44:30,945 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740 2024-12-05T00:44:30,945 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T00:44:30,945 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T00:44:30,945 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T00:44:30,946 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T00:44:30,947 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65157050, jitterRate=-0.02908429503440857}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T00:44:30,947 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T00:44:30,948 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733359470937Writing region info on filesystem at 1733359470937Initializing all the Stores at 1733359470937Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359470937Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359470938 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359470938Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359470938Cleaning up temporary data from old regions at 1733359470945 (+7 ms)Running coprocessor post-open hooks at 1733359470947 (+2 ms)Region opened successfully at 1733359470948 (+1 ms) 2024-12-05T00:44:30,953 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733359470885 2024-12-05T00:44:30,962 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T00:44:30,962 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T00:44:30,964 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:30,965 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fea72ea5c4b6,37427,1733359468844, state=OPEN 2024-12-05T00:44:30,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T00:44:30,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T00:44:30,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T00:44:30,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:30,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:30,976 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:30,976 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:30,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T00:44:30,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fea72ea5c4b6,37427,1733359468844 in 253 msec 2024-12-05T00:44:30,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T00:44:30,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 813 msec 2024-12-05T00:44:30,985 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T00:44:30,985 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T00:44:31,004 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:44:31,006 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,37427,1733359468844, seqNum=-1] 2024-12-05T00:44:31,021 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:44:31,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44377, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:44:31,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0470 sec 2024-12-05T00:44:31,041 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733359471041, completionTime=-1 2024-12-05T00:44:31,043 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-05T00:44:31,043 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T00:44:31,065 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T00:44:31,066 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733359531066 2024-12-05T00:44:31,066 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733359591066 2024-12-05T00:44:31,066 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-12-05T00:44:31,068 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-05T00:44:31,074 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,35373,1733359468043-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:31,074 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,35373,1733359468043-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:31,074 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,35373,1733359468043-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:31,076 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fea72ea5c4b6:35373, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:31,077 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:31,078 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:31,082 DEBUG [master/fea72ea5c4b6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T00:44:31,108 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.016sec 2024-12-05T00:44:31,110 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T00:44:31,111 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T00:44:31,112 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T00:44:31,113 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T00:44:31,113 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T00:44:31,114 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,35373,1733359468043-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T00:44:31,115 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,35373,1733359468043-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T00:44:31,116 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T00:44:31,117 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T00:44:31,117 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,35373,1733359468043-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:31,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885e29b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:44:31,124 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T00:44:31,124 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T00:44:31,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,35373,-1 for getting cluster id 2024-12-05T00:44:31,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:44:31,138 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '038e0f87-a58c-43fb-a8a9-e9242d89242c' 2024-12-05T00:44:31,141 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:44:31,141 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "038e0f87-a58c-43fb-a8a9-e9242d89242c" 2024-12-05T00:44:31,142 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ee5d66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:44:31,142 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,35373,-1] 2024-12-05T00:44:31,145 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:44:31,147 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:31,149 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:44:31,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1714ff62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:44:31,152 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:44:31,159 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,37427,1733359468844, seqNum=-1] 2024-12-05T00:44:31,159 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:44:31,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:44:31,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fea72ea5c4b6,35373,1733359468043 2024-12-05T00:44:31,196 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T00:44:31,204 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fea72ea5c4b6,35373,1733359468043 2024-12-05T00:44:31,207 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43e5817c 2024-12-05T00:44:31,208 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T00:44:31,210 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T00:44:31,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35373 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733359471191', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:44:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35373 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733359471191 2024-12-05T00:44:31,223 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733359471191 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T00:44:31,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35373 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733359471191" procId is: 4 2024-12-05T00:44:31,226 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:31,228 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733359471191 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T00:44:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:44:31,241 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733359471191-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7d89cb8a592e28bd4d4a07914f56b8a, NAME => 'testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733359471191', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b 2024-12-05T00:44:31,246 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733359471191-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:31,246 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733359471191-pool-0 {}] regionserver.HRegion(1722): Closing e7d89cb8a592e28bd4d4a07914f56b8a, disabling compactions & flushes 2024-12-05T00:44:31,246 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733359471191-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:31,247 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733359471191-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:31,247 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733359471191-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. after waiting 0 ms 2024-12-05T00:44:31,247 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733359471191-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:31,247 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733359471191-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:31,247 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733359471191-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7d89cb8a592e28bd4d4a07914f56b8a: Waiting for close lock at 1733359471246Disabling compacts and flushes for region at 1733359471246Disabling writes for close at 1733359471247 (+1 ms)Writing region close event to WAL at 1733359471247Closed at 1733359471247 2024-12-05T00:44:31,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733359471191 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T00:44:31,253 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733359471248"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733359471248"}]},"ts":"1733359471248"} 2024-12-05T00:44:31,257 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T00:44:31,258 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733359471191 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T00:44:31,260 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733359471191","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359471258"}]},"ts":"1733359471258"} 2024-12-05T00:44:31,263 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733359471191, state=ENABLING in hbase:meta 2024-12-05T00:44:31,264 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {fea72ea5c4b6=0} racks are {/default-rack=0} 2024-12-05T00:44:31,264 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T00:44:31,264 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T00:44:31,265 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T00:44:31,265 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T00:44:31,265 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T00:44:31,265 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T00:44:31,265 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T00:44:31,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733359471191, region=e7d89cb8a592e28bd4d4a07914f56b8a, ASSIGN}] 2024-12-05T00:44:31,267 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733359471191, region=e7d89cb8a592e28bd4d4a07914f56b8a, ASSIGN 2024-12-05T00:44:31,269 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733359471191, region=e7d89cb8a592e28bd4d4a07914f56b8a, ASSIGN; state=OFFLINE, location=fea72ea5c4b6,34765,1733359468969; forceNewPlan=false, retain=false 2024-12-05T00:44:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:44:31,422 INFO [fea72ea5c4b6:35373 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T00:44:31,423 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7d89cb8a592e28bd4d4a07914f56b8a, regionState=OPENING, regionLocation=fea72ea5c4b6,34765,1733359468969 2024-12-05T00:44:31,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733359471191, region=e7d89cb8a592e28bd4d4a07914f56b8a, ASSIGN because future has completed 2024-12-05T00:44:31,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d89cb8a592e28bd4d4a07914f56b8a, server=fea72ea5c4b6,34765,1733359468969}] 2024-12-05T00:44:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:44:31,581 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T00:44:31,584 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48321, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T00:44:31,589 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:31,589 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e7d89cb8a592e28bd4d4a07914f56b8a, NAME => 'testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:44:31,590 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733359471191 e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:31,590 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:31,590 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:31,590 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:31,591 INFO [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:31,594 INFO [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d89cb8a592e28bd4d4a07914f56b8a columnFamilyName family 2024-12-05T00:44:31,595 DEBUG [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:31,595 INFO [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] regionserver.HStore(327): Store=e7d89cb8a592e28bd4d4a07914f56b8a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:44:31,595 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:31,597 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:31,597 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:31,597 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:31,597 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:31,599 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:31,605 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:44:31,605 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e7d89cb8a592e28bd4d4a07914f56b8a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66603853, jitterRate=-0.007525250315666199}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T00:44:31,605 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:31,607 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e7d89cb8a592e28bd4d4a07914f56b8a: Running coprocessor pre-open hook at 1733359471590Writing region info on filesystem at 1733359471590Initializing all the Stores at 1733359471591 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359471591Cleaning up temporary data from old regions at 1733359471597 (+6 ms)Running coprocessor post-open hooks at 1733359471606 (+9 ms)Region opened successfully at 1733359471607 (+1 ms) 2024-12-05T00:44:31,608 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a., pid=6, masterSystemTime=1733359471580 2024-12-05T00:44:31,612 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:31,612 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:31,613 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7d89cb8a592e28bd4d4a07914f56b8a, regionState=OPEN, openSeqNum=2, regionLocation=fea72ea5c4b6,34765,1733359468969 2024-12-05T00:44:31,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d89cb8a592e28bd4d4a07914f56b8a, server=fea72ea5c4b6,34765,1733359468969 because future has completed 2024-12-05T00:44:31,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T00:44:31,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e7d89cb8a592e28bd4d4a07914f56b8a, server=fea72ea5c4b6,34765,1733359468969 in 191 msec 2024-12-05T00:44:31,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T00:44:31,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733359471191, region=e7d89cb8a592e28bd4d4a07914f56b8a, ASSIGN in 354 msec 2024-12-05T00:44:31,623 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733359471191 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T00:44:31,623 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733359471191","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359471623"}]},"ts":"1733359471623"} 2024-12-05T00:44:31,626 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733359471191, state=ENABLED in hbase:meta 2024-12-05T00:44:31,627 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733359471191 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T00:44:31,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733359471191 in 409 msec 2024-12-05T00:44:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:44:31,857 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733359471191 completed 2024-12-05T00:44:31,857 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733359471191 get assigned. Timeout = 60000ms 2024-12-05T00:44:31,860 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T00:44:31,866 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733359471191 assigned to meta. Checking AM states. 2024-12-05T00:44:31,866 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T00:44:31,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733359471191 assigned. 2024-12-05T00:44:31,875 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733359471191', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a., hostname=fea72ea5c4b6,34765,1733359468969, seqNum=2] 2024-12-05T00:44:31,877 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:44:31,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:44:31,890 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e7d89cb8a592e28bd4d4a07914f56b8a 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-05T00:44:31,912 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5d958552c2eb48aeb5eb95ebbdbdac11 2024-12-05T00:44:31,944 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/.tmp/family/5d958552c2eb48aeb5eb95ebbdbdac11 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-05T00:44:31,948 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/.tmp/family/5d958552c2eb48aeb5eb95ebbdbdac11 2024-12-05T00:44:31,975 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/.tmp/family/5d958552c2eb48aeb5eb95ebbdbdac11 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11 2024-12-05T00:44:31,977 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11, delay=938 ms 2024-12-05T00:44:31,978 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T00:44:31,983 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for e7d89cb8a592e28bd4d4a07914f56b8a in 94ms, sequenceid=6, compaction requested=false 2024-12-05T00:44:31,984 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733359471191' 2024-12-05T00:44:31,986 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e7d89cb8a592e28bd4d4a07914f56b8a: 2024-12-05T00:44:32,917 DEBUG [hfile-prefetch-1733359471977 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5d958552c2eb48aeb5eb95ebbdbdac11_0. 2024-12-05T00:44:32,922 DEBUG [hfile-prefetch-1733359471977 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5d958552c2eb48aeb5eb95ebbdbdac11_117. 2024-12-05T00:44:32,923 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11, with total blocks 2, and data blocks 1 2024-12-05T00:44:32,923 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-05T00:44:32,923 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-05T00:44:32,924 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11. Will sleep 100ms and try the verification again. 2024-12-05T00:44:33,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35373 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=e7d89cb8a592e28bd4d4a07914f56b8a, source=fea72ea5c4b6,34765,1733359468969, destination=fea72ea5c4b6,37427,1733359468844, warming up region on fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:33,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35373 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=e7d89cb8a592e28bd4d4a07914f56b8a, source=fea72ea5c4b6,34765,1733359468969, destination=fea72ea5c4b6,37427,1733359468844, running balancer 2024-12-05T00:44:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35373 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733359471191, region=e7d89cb8a592e28bd4d4a07914f56b8a, REOPEN/MOVE 2024-12-05T00:44:33,004 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733359471191, region=e7d89cb8a592e28bd4d4a07914f56b8a, REOPEN/MOVE 2024-12-05T00:44:33,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37427 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37427 {}] regionserver.HRegion(7855): Warmup {ENCODED => e7d89cb8a592e28bd4d4a07914f56b8a, NAME => 'testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:44:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37427 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:33,005 INFO [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,006 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e7d89cb8a592e28bd4d4a07914f56b8a, regionState=CLOSING, regionLocation=fea72ea5c4b6,34765,1733359468969 2024-12-05T00:44:33,006 INFO [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d89cb8a592e28bd4d4a07914f56b8a columnFamilyName family 2024-12-05T00:44:33,007 DEBUG [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:33,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733359471191, region=e7d89cb8a592e28bd4d4a07914f56b8a, REOPEN/MOVE because future has completed 2024-12-05T00:44:33,010 DEBUG [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11 2024-12-05T00:44:33,010 INFO [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] regionserver.HStore(327): Store=e7d89cb8a592e28bd4d4a07914f56b8a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:44:33,010 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-05T00:44:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37427 {}] regionserver.HRegion(1722): Closing e7d89cb8a592e28bd4d4a07914f56b8a, disabling compactions & flushes 2024-12-05T00:44:33,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37427 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37427 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37427 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. after waiting 0 ms 2024-12-05T00:44:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37427 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7d89cb8a592e28bd4d4a07914f56b8a, server=fea72ea5c4b6,34765,1733359468969}] 2024-12-05T00:44:33,012 DEBUG [StoreFileCloser-e7d89cb8a592e28bd4d4a07914f56b8a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11 2024-12-05T00:44:33,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37427 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37427 {}] regionserver.HRegion(1676): Region close journal for e7d89cb8a592e28bd4d4a07914f56b8a: Waiting for close lock at 1733359473010Disabling compacts and flushes for region at 1733359473010Disabling writes for close at 1733359473010Writing region close event to WAL at 1733359473013 (+3 ms)Closed at 1733359473013 2024-12-05T00:44:33,014 WARN [hfile-prefetch-1733359471977 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-05T00:44:33,015 DEBUG [hfile-prefetch-1733359471977 {}] hfile.PrefetchExecutor(121): Prefetch completed for 5d958552c2eb48aeb5eb95ebbdbdac11 2024-12-05T00:44:33,029 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T00:44:33,029 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T00:44:33,029 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T00:44:33,029 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T00:44:33,044 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T00:44:33,044 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733359473029 2024-12-05T00:44:33,044 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T00:44:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35373 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-05T00:44:33,170 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,170 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-05T00:44:33,171 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing e7d89cb8a592e28bd4d4a07914f56b8a, disabling compactions & flushes 2024-12-05T00:44:33,171 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,172 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,172 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. after waiting 0 ms 2024-12-05T00:44:33,172 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,174 DEBUG [StoreFileCloser-e7d89cb8a592e28bd4d4a07914f56b8a-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 5d958552c2eb48aeb5eb95ebbdbdac11, starting offset: 0, end offset: 9223372036854775807 2024-12-05T00:44:33,176 DEBUG [StoreFileCloser-e7d89cb8a592e28bd4d4a07914f56b8a-family-1 {}] bucket.BucketCache(856): removed key 5d958552c2eb48aeb5eb95ebbdbdac11_0 from back map with offset lock 2094080 in the evict process 2024-12-05T00:44:33,182 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T00:44:33,183 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,183 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for e7d89cb8a592e28bd4d4a07914f56b8a: Waiting for close lock at 1733359473171Running coprocessor pre-close hooks at 1733359473171Disabling compacts and flushes for region at 1733359473171Disabling writes for close at 1733359473172 (+1 ms)Writing region close event to WAL at 1733359473177 (+5 ms)Running coprocessor post-close hooks at 1733359473182 (+5 ms)Closed at 1733359473183 (+1 ms) 2024-12-05T00:44:33,183 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding e7d89cb8a592e28bd4d4a07914f56b8a move to fea72ea5c4b6,37427,1733359468844 record at close sequenceid=6 2024-12-05T00:44:33,186 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,187 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e7d89cb8a592e28bd4d4a07914f56b8a, regionState=CLOSED 2024-12-05T00:44:33,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7d89cb8a592e28bd4d4a07914f56b8a, server=fea72ea5c4b6,34765,1733359468969 because future has completed 2024-12-05T00:44:33,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T00:44:33,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure e7d89cb8a592e28bd4d4a07914f56b8a, server=fea72ea5c4b6,34765,1733359468969 in 181 msec 2024-12-05T00:44:33,193 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733359471191, region=e7d89cb8a592e28bd4d4a07914f56b8a, REOPEN/MOVE; state=CLOSED, location=fea72ea5c4b6,37427,1733359468844; forceNewPlan=false, retain=false 2024-12-05T00:44:33,194 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T00:44:33,194 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T00:44:33,194 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T00:44:33,194 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T00:44:33,196 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T00:44:33,196 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733359473194 2024-12-05T00:44:33,196 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T00:44:33,344 INFO [fea72ea5c4b6:35373 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T00:44:33,345 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e7d89cb8a592e28bd4d4a07914f56b8a, regionState=OPENING, regionLocation=fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:33,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733359471191, region=e7d89cb8a592e28bd4d4a07914f56b8a, REOPEN/MOVE because future has completed 2024-12-05T00:44:33,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d89cb8a592e28bd4d4a07914f56b8a, server=fea72ea5c4b6,37427,1733359468844}] 2024-12-05T00:44:33,510 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,511 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e7d89cb8a592e28bd4d4a07914f56b8a, NAME => 'testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:44:33,512 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733359471191 e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,512 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:33,513 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,513 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,515 INFO [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,516 INFO [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d89cb8a592e28bd4d4a07914f56b8a columnFamilyName family 2024-12-05T00:44:33,516 DEBUG [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:33,518 DEBUG [StoreFileOpener-e7d89cb8a592e28bd4d4a07914f56b8a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11, delay=993 ms 2024-12-05T00:44:33,519 DEBUG [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11 2024-12-05T00:44:33,519 INFO [StoreOpener-e7d89cb8a592e28bd4d4a07914f56b8a-1 {}] regionserver.HStore(327): Store=e7d89cb8a592e28bd4d4a07914f56b8a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:44:33,520 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,520 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,521 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,521 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,521 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,526 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,526 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened e7d89cb8a592e28bd4d4a07914f56b8a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61275908, jitterRate=-0.08691781759262085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T00:44:33,526 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,527 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e7d89cb8a592e28bd4d4a07914f56b8a: Running coprocessor pre-open hook at 1733359473513Writing region info on filesystem at 1733359473513Initializing all the Stores at 1733359473514 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359473514Cleaning up temporary data from old regions at 1733359473521 (+7 ms)Running coprocessor post-open hooks at 1733359473526 (+5 ms)Region opened successfully at 1733359473527 (+1 ms) 2024-12-05T00:44:33,528 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a., pid=9, masterSystemTime=1733359473505 2024-12-05T00:44:33,531 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,531 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,532 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e7d89cb8a592e28bd4d4a07914f56b8a, regionState=OPEN, openSeqNum=10, regionLocation=fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:33,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d89cb8a592e28bd4d4a07914f56b8a, server=fea72ea5c4b6,37427,1733359468844 because future has completed 2024-12-05T00:44:33,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-05T00:44:33,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure e7d89cb8a592e28bd4d4a07914f56b8a, server=fea72ea5c4b6,37427,1733359468844 in 185 msec 2024-12-05T00:44:33,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733359471191, region=e7d89cb8a592e28bd4d4a07914f56b8a, REOPEN/MOVE in 535 msec 2024-12-05T00:44:33,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T00:44:33,557 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:44:33,558 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:33,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:33,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:33,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:44:33,562 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T00:44:33,562 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-12-05T00:44:33,562 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fea72ea5c4b6,35373,1733359468043 2024-12-05T00:44:33,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:33,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:33,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:33,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:33,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:33,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:33,614 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T00:44:33,615 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:44:33,615 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:33,615 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:33,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:33,615 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:33,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:33,617 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fea72ea5c4b6,37427,1733359468844' ***** 2024-12-05T00:44:33,617 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T00:44:33,617 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fea72ea5c4b6,34765,1733359468969' ***** 2024-12-05T00:44:33,617 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T00:44:33,617 INFO [RS:0;fea72ea5c4b6:37427 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T00:44:33,618 INFO [RS:1;fea72ea5c4b6:34765 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T00:44:33,618 INFO [RS:0;fea72ea5c4b6:37427 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/bucket.persistence 2024-12-05T00:44:33,618 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T00:44:33,618 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T00:44:33,618 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T00:44:33,618 INFO [RS:1;fea72ea5c4b6:34765 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/bucket.persistence 2024-12-05T00:44:33,618 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T00:44:33,618 INFO [RS:0;fea72ea5c4b6:37427 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T00:44:33,618 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T00:44:33,618 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T00:44:33,618 INFO [RS:1;fea72ea5c4b6:34765 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T00:44:33,618 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T00:44:33,619 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T00:44:33,628 DEBUG [RS:0;fea72ea5c4b6:37427 {}] bucket.BucketCache(1487): Thread RS:0;fea72ea5c4b6:37427 started persisting bucket cache to file 2024-12-05T00:44:33,629 DEBUG [RS:0;fea72ea5c4b6:37427 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T00:44:33,629 DEBUG [RS:1;fea72ea5c4b6:34765 {}] bucket.BucketCache(1487): Thread RS:1;fea72ea5c4b6:34765 started persisting bucket cache to file 2024-12-05T00:44:33,629 DEBUG [RS:0;fea72ea5c4b6:37427 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T00:44:33,629 DEBUG [RS:1;fea72ea5c4b6:34765 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T00:44:33,629 DEBUG [RS:1;fea72ea5c4b6:34765 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T00:44:33,631 DEBUG [RS:1;fea72ea5c4b6:34765 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T00:44:33,631 DEBUG [RS:0;fea72ea5c4b6:37427 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T00:44:33,631 DEBUG [RS:1;fea72ea5c4b6:34765 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733359473629 2024-12-05T00:44:33,631 DEBUG [RS:0;fea72ea5c4b6:37427 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733359473628 2024-12-05T00:44:33,631 DEBUG [RS:1;fea72ea5c4b6:34765 {}] bucket.BucketCache(1509): Thread RS:1;fea72ea5c4b6:34765 finished persisting bucket cache to file, renaming 2024-12-05T00:44:33,631 DEBUG [RS:0;fea72ea5c4b6:37427 {}] bucket.BucketCache(1509): Thread RS:0;fea72ea5c4b6:37427 finished persisting bucket cache to file, renaming 2024-12-05T00:44:33,632 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T00:44:33,632 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T00:44:33,632 INFO [RS:1;fea72ea5c4b6:34765 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T00:44:33,632 INFO [RS:0;fea72ea5c4b6:37427 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T00:44:33,632 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T00:44:33,632 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T00:44:33,632 INFO [RS:1;fea72ea5c4b6:34765 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T00:44:33,632 INFO [RS:0;fea72ea5c4b6:37427 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T00:44:33,632 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer(959): stopping server fea72ea5c4b6,34765,1733359468969 2024-12-05T00:44:33,632 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(3091): Received CLOSE for e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,632 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T00:44:33,632 INFO [RS:1;fea72ea5c4b6:34765 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fea72ea5c4b6:34765. 2024-12-05T00:44:33,632 DEBUG [RS:1;fea72ea5c4b6:34765 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:33,632 DEBUG [RS:1;fea72ea5c4b6:34765 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:33,633 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(959): stopping server fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:33,633 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer(976): stopping server fea72ea5c4b6,34765,1733359468969; all regions closed. 2024-12-05T00:44:33,633 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T00:44:33,633 INFO [RS:0;fea72ea5c4b6:37427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fea72ea5c4b6:37427. 2024-12-05T00:44:33,633 DEBUG [RS:0;fea72ea5c4b6:37427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:33,633 DEBUG [RS:0;fea72ea5c4b6:37427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:33,633 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T00:44:33,633 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T00:44:33,633 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T00:44:33,633 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T00:44:33,633 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T00:44:33,633 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e7d89cb8a592e28bd4d4a07914f56b8a, disabling compactions & flushes 2024-12-05T00:44:33,633 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,634 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,634 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. after waiting 0 ms 2024-12-05T00:44:33,634 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-05T00:44:33,634 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,634 DEBUG [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(1325): Online Regions={e7d89cb8a592e28bd4d4a07914f56b8a=testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a., 1588230740=hbase:meta,,1.1588230740} 2024-12-05T00:44:33,634 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T00:44:33,634 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T00:44:33,634 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T00:44:33,634 DEBUG [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e7d89cb8a592e28bd4d4a07914f56b8a 2024-12-05T00:44:33,634 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T00:44:33,634 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T00:44:33,634 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T00:44:33,634 DEBUG [StoreFileCloser-e7d89cb8a592e28bd4d4a07914f56b8a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/family/5d958552c2eb48aeb5eb95ebbdbdac11 2024-12-05T00:44:33,635 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-05T00:44:33,637 DEBUG [RS:1;fea72ea5c4b6:34765 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs 2024-12-05T00:44:33,637 INFO [RS:1;fea72ea5c4b6:34765 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fea72ea5c4b6%2C34765%2C1733359468969:(num 1733359470446) 2024-12-05T00:44:33,637 DEBUG [RS:1;fea72ea5c4b6:34765 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:33,637 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:33,637 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T00:44:33,638 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.ChoreService(370): Chore service for: regionserver/fea72ea5c4b6:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T00:44:33,638 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T00:44:33,638 INFO [regionserver/fea72ea5c4b6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T00:44:33,638 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T00:44:33,638 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T00:44:33,638 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T00:44:33,639 INFO [RS:1;fea72ea5c4b6:34765 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34765 2024-12-05T00:44:33,639 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnRegionMove1733359471191/e7d89cb8a592e28bd4d4a07914f56b8a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-05T00:44:33,639 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,639 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e7d89cb8a592e28bd4d4a07914f56b8a: Waiting for close lock at 1733359473633Running coprocessor pre-close hooks at 1733359473633Disabling compacts and flushes for region at 1733359473633Disabling writes for close at 1733359473634 (+1 ms)Writing region close event to WAL at 1733359473635 (+1 ms)Running coprocessor post-close hooks at 1733359473639 (+4 ms)Closed at 1733359473639 2024-12-05T00:44:33,640 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a. 2024-12-05T00:44:33,644 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0a5df2ed446c4f7ba15fb804e5dcae5c 2024-12-05T00:44:33,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fea72ea5c4b6,34765,1733359468969 2024-12-05T00:44:33,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T00:44:33,656 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T00:44:33,665 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/info/0a5df2ed446c4f7ba15fb804e5dcae5c is 205, key is testBlockEvictionOnRegionMove1733359471191,,1733359471212.e7d89cb8a592e28bd4d4a07914f56b8a./info:regioninfo/1733359473532/Put/seqid=0 2024-12-05T00:44:33,666 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fea72ea5c4b6,34765,1733359468969] 2024-12-05T00:44:33,667 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/info/0a5df2ed446c4f7ba15fb804e5dcae5c 2024-12-05T00:44:33,676 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 331391af87b34790b5c44782d6bf1797 2024-12-05T00:44:33,676 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fea72ea5c4b6,34765,1733359468969 already deleted, retry=false 2024-12-05T00:44:33,677 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fea72ea5c4b6,34765,1733359468969 expired; onlineServers=1 2024-12-05T00:44:33,688 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/ns/331391af87b34790b5c44782d6bf1797 is 43, key is default/ns:d/1733359471028/Put/seqid=0 2024-12-05T00:44:33,689 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/ns/331391af87b34790b5c44782d6bf1797 2024-12-05T00:44:33,693 INFO [regionserver/fea72ea5c4b6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:33,694 INFO [regionserver/fea72ea5c4b6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:33,697 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9bc910d5f36d49c6b5a5fbdeb2edd938 2024-12-05T00:44:33,713 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/table/9bc910d5f36d49c6b5a5fbdeb2edd938 is 78, key is testBlockEvictionOnRegionMove1733359471191/table:state/1733359471623/Put/seqid=0 2024-12-05T00:44:33,714 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/table/9bc910d5f36d49c6b5a5fbdeb2edd938 2024-12-05T00:44:33,718 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/info/0a5df2ed446c4f7ba15fb804e5dcae5c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/info/0a5df2ed446c4f7ba15fb804e5dcae5c 2024-12-05T00:44:33,721 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/info/0a5df2ed446c4f7ba15fb804e5dcae5c, delay=923 ms 2024-12-05T00:44:33,722 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/info/0a5df2ed446c4f7ba15fb804e5dcae5c, entries=15, sequenceid=15, filesize=7.6 K 2024-12-05T00:44:33,723 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/ns/331391af87b34790b5c44782d6bf1797 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/ns/331391af87b34790b5c44782d6bf1797 2024-12-05T00:44:33,725 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/ns/331391af87b34790b5c44782d6bf1797, delay=932 ms 2024-12-05T00:44:33,727 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/ns/331391af87b34790b5c44782d6bf1797, entries=2, sequenceid=15, filesize=5.0 K 2024-12-05T00:44:33,728 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/table/9bc910d5f36d49c6b5a5fbdeb2edd938 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/table/9bc910d5f36d49c6b5a5fbdeb2edd938 2024-12-05T00:44:33,729 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/table/9bc910d5f36d49c6b5a5fbdeb2edd938, delay=966 ms 2024-12-05T00:44:33,730 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/table/9bc910d5f36d49c6b5a5fbdeb2edd938, entries=2, sequenceid=15, filesize=5.3 K 2024-12-05T00:44:33,731 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=15, compaction requested=false 2024-12-05T00:44:33,732 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T00:44:33,733 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/info/0a5df2ed446c4f7ba15fb804e5dcae5c 2024-12-05T00:44:33,734 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/ns/331391af87b34790b5c44782d6bf1797 2024-12-05T00:44:33,735 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/table/9bc910d5f36d49c6b5a5fbdeb2edd938 2024-12-05T00:44:33,741 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-05T00:44:33,742 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T00:44:33,742 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T00:44:33,742 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733359473634Running coprocessor pre-close hooks at 1733359473634Disabling compacts and flushes for region at 1733359473634Disabling writes for close at 1733359473634Obtaining lock to block concurrent updates at 1733359473635 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733359473635Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733359473635Flushing stores of hbase:meta,,1.1588230740 at 1733359473635Flushing 1588230740/info: creating writer at 1733359473636 (+1 ms)Flushing 1588230740/info: appending metadata at 1733359473663 (+27 ms)Flushing 1588230740/info: closing flushed file at 1733359473663Flushing 1588230740/ns: creating writer at 1733359473670 (+7 ms)Flushing 1588230740/ns: appending metadata at 1733359473687 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733359473687Flushing 1588230740/table: creating writer at 1733359473691 (+4 ms)Flushing 1588230740/table: appending metadata at 1733359473713 (+22 ms)Flushing 1588230740/table: closing flushed file at 1733359473713Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@446121f0: reopening flushed file at 1733359473718 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ce49568: reopening flushed file at 1733359473723 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cfd4283: reopening flushed file at 1733359473727 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=15, compaction requested=false at 1733359473732 (+5 ms)Writing region close event to WAL at 1733359473735 (+3 ms)Running coprocessor post-close hooks at 1733359473741 (+6 ms)Closed at 1733359473742 (+1 ms) 2024-12-05T00:44:33,743 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T00:44:33,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:33,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34765-0x101a2f278c30002, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:33,767 INFO [RS:1;fea72ea5c4b6:34765 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T00:44:33,768 INFO [RS:1;fea72ea5c4b6:34765 {}] regionserver.HRegionServer(1031): Exiting; stopping=fea72ea5c4b6,34765,1733359468969; zookeeper connection closed. 2024-12-05T00:44:33,768 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@77de8fe1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@77de8fe1 2024-12-05T00:44:33,835 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(976): stopping server fea72ea5c4b6,37427,1733359468844; all regions closed. 2024-12-05T00:44:33,843 DEBUG [RS:0;fea72ea5c4b6:37427 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs 2024-12-05T00:44:33,843 INFO [RS:0;fea72ea5c4b6:37427 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fea72ea5c4b6%2C37427%2C1733359468844.meta:.meta(num 1733359470907) 2024-12-05T00:44:33,851 DEBUG [RS:0;fea72ea5c4b6:37427 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs 2024-12-05T00:44:33,851 INFO [RS:0;fea72ea5c4b6:37427 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fea72ea5c4b6%2C37427%2C1733359468844:(num 1733359470446) 2024-12-05T00:44:33,851 DEBUG [RS:0;fea72ea5c4b6:37427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:33,851 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:33,851 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T00:44:33,852 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.ChoreService(370): Chore service for: regionserver/fea72ea5c4b6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T00:44:33,852 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T00:44:33,853 INFO [regionserver/fea72ea5c4b6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T00:44:33,853 INFO [RS:0;fea72ea5c4b6:37427 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37427 2024-12-05T00:44:33,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fea72ea5c4b6,37427,1733359468844 2024-12-05T00:44:33,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T00:44:33,867 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T00:44:33,868 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fea72ea5c4b6,37427,1733359468844] 2024-12-05T00:44:33,890 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fea72ea5c4b6,37427,1733359468844 already deleted, retry=false 2024-12-05T00:44:33,891 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fea72ea5c4b6,37427,1733359468844 expired; onlineServers=0 2024-12-05T00:44:33,891 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fea72ea5c4b6,35373,1733359468043' ***** 2024-12-05T00:44:33,891 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T00:44:33,891 INFO [M:0;fea72ea5c4b6:35373 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T00:44:33,892 INFO [M:0;fea72ea5c4b6:35373 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T00:44:33,892 DEBUG [M:0;fea72ea5c4b6:35373 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T00:44:33,892 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T00:44:33,892 DEBUG [M:0;fea72ea5c4b6:35373 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T00:44:33,892 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.large.0-1733359470091 {}] cleaner.HFileCleaner(306): Exit Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.large.0-1733359470091,5,FailOnTimeoutGroup] 2024-12-05T00:44:33,892 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.small.0-1733359470092 {}] cleaner.HFileCleaner(306): Exit Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.small.0-1733359470092,5,FailOnTimeoutGroup] 2024-12-05T00:44:33,893 INFO [M:0;fea72ea5c4b6:35373 {}] hbase.ChoreService(370): Chore service for: master/fea72ea5c4b6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T00:44:33,893 INFO [M:0;fea72ea5c4b6:35373 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T00:44:33,893 DEBUG [M:0;fea72ea5c4b6:35373 {}] master.HMaster(1795): Stopping service threads 2024-12-05T00:44:33,894 INFO [M:0;fea72ea5c4b6:35373 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T00:44:33,894 INFO [M:0;fea72ea5c4b6:35373 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T00:44:33,895 INFO [M:0;fea72ea5c4b6:35373 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T00:44:33,895 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T00:44:33,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T00:44:33,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:33,901 DEBUG [M:0;fea72ea5c4b6:35373 {}] zookeeper.ZKUtil(347): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T00:44:33,902 WARN [M:0;fea72ea5c4b6:35373 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T00:44:33,902 INFO [M:0;fea72ea5c4b6:35373 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/.lastflushedseqids 2024-12-05T00:44:33,937 INFO [M:0;fea72ea5c4b6:35373 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T00:44:33,938 INFO [M:0;fea72ea5c4b6:35373 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T00:44:33,938 DEBUG [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T00:44:33,938 INFO [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:33,938 DEBUG [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:33,938 DEBUG [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T00:44:33,938 DEBUG [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:33,939 INFO [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-12-05T00:44:33,958 DEBUG [M:0;fea72ea5c4b6:35373 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cc71db8d825444faa03cc388e786ab86 2024-12-05T00:44:33,978 DEBUG [M:0;fea72ea5c4b6:35373 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc71db8d825444faa03cc388e786ab86 is 82, key is hbase:meta,,1/info:regioninfo/1733359470963/Put/seqid=0 2024-12-05T00:44:33,979 INFO [M:0;fea72ea5c4b6:35373 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc71db8d825444faa03cc388e786ab86 2024-12-05T00:44:33,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:33,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37427-0x101a2f278c30001, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:33,981 INFO [RS:0;fea72ea5c4b6:37427 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T00:44:33,981 INFO [RS:0;fea72ea5c4b6:37427 {}] regionserver.HRegionServer(1031): Exiting; stopping=fea72ea5c4b6,37427,1733359468844; zookeeper connection closed. 2024-12-05T00:44:33,992 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@43932b71 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@43932b71 2024-12-05T00:44:33,993 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-05T00:44:34,021 DEBUG [M:0;fea72ea5c4b6:35373 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 42f459647abe4c1aae37f84f45c65b7d 2024-12-05T00:44:34,042 DEBUG [M:0;fea72ea5c4b6:35373 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/42f459647abe4c1aae37f84f45c65b7d is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733359471628/Put/seqid=0 2024-12-05T00:44:34,043 INFO [M:0;fea72ea5c4b6:35373 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/42f459647abe4c1aae37f84f45c65b7d 2024-12-05T00:44:34,054 DEBUG [M:0;fea72ea5c4b6:35373 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dd1f61f912c94220ae61f4478972a60a 2024-12-05T00:44:34,073 DEBUG [M:0;fea72ea5c4b6:35373 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dd1f61f912c94220ae61f4478972a60a is 69, key is fea72ea5c4b6,34765,1733359468969/rs:state/1733359470164/Put/seqid=0 2024-12-05T00:44:34,073 INFO [M:0;fea72ea5c4b6:35373 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dd1f61f912c94220ae61f4478972a60a 2024-12-05T00:44:34,076 DEBUG [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc71db8d825444faa03cc388e786ab86 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc71db8d825444faa03cc388e786ab86 2024-12-05T00:44:34,078 INFO [M:0;fea72ea5c4b6:35373 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc71db8d825444faa03cc388e786ab86, entries=8, sequenceid=84, filesize=5.5 K 2024-12-05T00:44:34,079 DEBUG [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/42f459647abe4c1aae37f84f45c65b7d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/42f459647abe4c1aae37f84f45c65b7d 2024-12-05T00:44:34,083 INFO [M:0;fea72ea5c4b6:35373 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/42f459647abe4c1aae37f84f45c65b7d, entries=9, sequenceid=84, filesize=6.5 K 2024-12-05T00:44:34,083 DEBUG [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dd1f61f912c94220ae61f4478972a60a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dd1f61f912c94220ae61f4478972a60a 2024-12-05T00:44:34,086 INFO [M:0;fea72ea5c4b6:35373 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dd1f61f912c94220ae61f4478972a60a, entries=2, sequenceid=84, filesize=5.1 K 2024-12-05T00:44:34,087 INFO [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 149ms, sequenceid=84, compaction requested=false 2024-12-05T00:44:34,103 INFO [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:34,103 DEBUG [M:0;fea72ea5c4b6:35373 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733359473938Disabling compacts and flushes for region at 1733359473938Disabling writes for close at 1733359473938Obtaining lock to block concurrent updates at 1733359473939 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733359473939Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733359473940 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733359473941 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733359473941Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733359473977 (+36 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733359473977Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733359473986 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733359474041 (+55 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733359474041Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733359474046 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733359474072 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733359474072Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60d10f2f: reopening flushed file at 1733359474076 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d39f40: reopening flushed file at 1733359474079 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be31822: reopening flushed file at 1733359474083 (+4 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 149ms, sequenceid=84, compaction requested=false at 1733359474087 (+4 ms)Writing region close event to WAL at 1733359474103 (+16 ms)Closed at 1733359474103 2024-12-05T00:44:34,116 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T00:44:34,116 INFO [M:0;fea72ea5c4b6:35373 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T00:44:34,116 INFO [M:0;fea72ea5c4b6:35373 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35373 2024-12-05T00:44:34,116 INFO [M:0;fea72ea5c4b6:35373 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T00:44:34,230 INFO [M:0;fea72ea5c4b6:35373 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T00:44:34,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:34,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35373-0x101a2f278c30000, quorum=127.0.0.1:57216, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:34,236 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T00:44:34,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-05T00:44:34,266 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:34,295 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1733359473721 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733359473725 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/fea72ea5c4b6:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1733359471977 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/fea72ea5c4b6:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1733359473518 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/fea72ea5c4b6:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/fea72ea5c4b6:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=310 (was 294) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10193 (was 10562) 2024-12-05T00:44:34,308 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=310, ProcessCount=11, AvailableMemoryMB=10192 2024-12-05T00:44:34,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/hadoop-log-dir so I do NOT create it in target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79 2024-12-05T00:44:34,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/hadoop-tmp-dir so I do NOT create it in target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79 2024-12-05T00:44:34,308 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:34,316 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/cluster_73d92d6a-25c1-c258-341b-135327833110, deleteOnExit=true 2024-12-05T00:44:34,325 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/cluster_73d92d6a-25c1-c258-341b-135327833110/zookeeper_0, clientPort=50590, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/cluster_73d92d6a-25c1-c258-341b-135327833110/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/cluster_73d92d6a-25c1-c258-341b-135327833110/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T00:44:34,329 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50590 2024-12-05T00:44:34,352 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b with version=8 2024-12-05T00:44:34,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/hbase-staging 2024-12-05T00:44:34,355 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fea72ea5c4b6:0 server-side Connection retries=45 2024-12-05T00:44:34,355 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:34,356 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:34,356 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T00:44:34,356 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:34,356 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T00:44:34,356 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T00:44:34,356 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T00:44:34,365 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46335 2024-12-05T00:44:34,367 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46335 connecting to ZooKeeper ensemble=127.0.0.1:50590 2024-12-05T00:44:34,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:463350x0, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T00:44:34,425 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46335-0x101a2f2923b0000 connected 2024-12-05T00:44:34,515 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:34,515 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:34,517 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:34,518 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b, hbase.cluster.distributed=false 2024-12-05T00:44:34,520 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T00:44:34,521 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46335 2024-12-05T00:44:34,521 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46335 2024-12-05T00:44:34,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46335 2024-12-05T00:44:34,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46335 2024-12-05T00:44:34,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46335 2024-12-05T00:44:34,540 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fea72ea5c4b6:0 server-side Connection retries=45 2024-12-05T00:44:34,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:34,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:34,540 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T00:44:34,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:34,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T00:44:34,540 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T00:44:34,540 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T00:44:34,541 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32821 2024-12-05T00:44:34,542 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32821 connecting to ZooKeeper ensemble=127.0.0.1:50590 2024-12-05T00:44:34,543 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:34,543 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:34,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328210x0, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T00:44:34,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32821-0x101a2f2923b0001 connected 2024-12-05T00:44:34,559 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:34,560 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T00:44:34,562 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.cache 2024-12-05T00:44:34,562 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T00:44:34,563 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T00:44:34,563 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T00:44:34,563 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T00:44:34,563 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T00:44:34,564 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T00:44:34,564 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.persistence is complete. 2024-12-05T00:44:34,564 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T00:44:34,568 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T00:44:34,569 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T00:44:34,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T00:44:34,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32821 2024-12-05T00:44:34,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32821 2024-12-05T00:44:34,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32821 2024-12-05T00:44:34,575 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32821 2024-12-05T00:44:34,575 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32821 2024-12-05T00:44:34,593 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fea72ea5c4b6:0 server-side Connection retries=45 2024-12-05T00:44:34,593 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:34,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:34,594 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T00:44:34,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:34,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T00:44:34,594 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T00:44:34,594 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T00:44:34,595 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38585 2024-12-05T00:44:34,597 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38585 connecting to ZooKeeper ensemble=127.0.0.1:50590 2024-12-05T00:44:34,598 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:34,598 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:34,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385850x0, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T00:44:34,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:385850x0, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:34,610 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38585-0x101a2f2923b0002 connected 2024-12-05T00:44:34,611 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T00:44:34,612 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.cache 2024-12-05T00:44:34,612 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T00:44:34,612 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T00:44:34,613 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T00:44:34,613 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T00:44:34,613 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T00:44:34,613 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T00:44:34,613 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T00:44:34,613 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T00:44:34,613 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.persistence is complete. 2024-12-05T00:44:34,614 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T00:44:34,616 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T00:44:34,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38585 2024-12-05T00:44:34,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38585 2024-12-05T00:44:34,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38585 2024-12-05T00:44:34,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38585 2024-12-05T00:44:34,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38585 2024-12-05T00:44:34,636 DEBUG [M:0;fea72ea5c4b6:46335 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fea72ea5c4b6:46335 2024-12-05T00:44:34,636 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fea72ea5c4b6,46335,1733359474355 2024-12-05T00:44:34,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:34,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:34,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:34,648 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fea72ea5c4b6,46335,1733359474355 2024-12-05T00:44:34,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:34,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T00:44:34,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T00:44:34,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:34,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:34,657 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T00:44:34,657 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fea72ea5c4b6,46335,1733359474355 from backup master directory 2024-12-05T00:44:34,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:34,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fea72ea5c4b6,46335,1733359474355 2024-12-05T00:44:34,773 WARN [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T00:44:34,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:34,773 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fea72ea5c4b6,46335,1733359474355 2024-12-05T00:44:34,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:44:34,774 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/hbase.id] with ID: 2afb9446-ad7f-49d7-9e08-887edae3137d 2024-12-05T00:44:34,774 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/.tmp/hbase.id 2024-12-05T00:44:34,778 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/hbase.id] 2024-12-05T00:44:34,823 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:34,823 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T00:44:34,825 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-05T00:44:34,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:34,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:34,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:34,862 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:44:34,864 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T00:44:34,864 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:44:34,889 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store 2024-12-05T00:44:34,898 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:34,898 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T00:44:34,898 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:34,898 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:34,898 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T00:44:34,898 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:34,898 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:34,898 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733359474898Disabling compacts and flushes for region at 1733359474898Disabling writes for close at 1733359474898Writing region close event to WAL at 1733359474898Closed at 1733359474898 2024-12-05T00:44:34,899 WARN [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/.initializing 2024-12-05T00:44:34,899 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/WALs/fea72ea5c4b6,46335,1733359474355 2024-12-05T00:44:34,906 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T00:44:34,910 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fea72ea5c4b6%2C46335%2C1733359474355, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/WALs/fea72ea5c4b6,46335,1733359474355, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/oldWALs, maxLogs=10 2024-12-05T00:44:34,927 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/WALs/fea72ea5c4b6,46335,1733359474355/fea72ea5c4b6%2C46335%2C1733359474355.1733359474910 2024-12-05T00:44:34,927 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T00:44:34,927 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:44:34,927 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:34,927 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,927 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,937 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T00:44:34,938 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:34,938 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:34,938 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,943 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T00:44:34,943 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:34,944 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:44:34,944 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,948 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T00:44:34,948 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:34,949 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:44:34,949 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,953 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T00:44:34,953 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:34,954 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:44:34,954 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,955 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,955 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,955 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,955 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,955 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T00:44:34,956 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:44:34,963 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:44:34,964 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70480147, jitterRate=0.05023603141307831}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T00:44:34,965 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733359474928Initializing all the Stores at 1733359474928Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359474928Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359474928Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359474928Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359474928Cleaning up temporary data from old regions at 1733359474955 (+27 ms)Region opened successfully at 1733359474965 (+10 ms) 2024-12-05T00:44:34,966 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T00:44:34,971 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38ab87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fea72ea5c4b6/172.17.0.2:0 2024-12-05T00:44:34,973 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T00:44:34,973 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T00:44:34,973 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T00:44:34,974 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T00:44:34,974 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T00:44:34,975 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-05T00:44:34,975 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T00:44:34,979 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T00:44:34,981 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T00:44:34,993 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T00:44:34,993 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T00:44:34,994 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T00:44:35,043 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T00:44:35,044 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T00:44:35,045 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T00:44:35,067 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T00:44:35,069 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T00:44:35,077 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T00:44:35,081 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T00:44:35,148 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T00:44:35,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:35,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:35,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:35,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,204 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fea72ea5c4b6,46335,1733359474355, sessionid=0x101a2f2923b0000, setting cluster-up flag (Was=false) 2024-12-05T00:44:35,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,256 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T00:44:35,258 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fea72ea5c4b6,46335,1733359474355 2024-12-05T00:44:35,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,309 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T00:44:35,310 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fea72ea5c4b6,46335,1733359474355 2024-12-05T00:44:35,311 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T00:44:35,313 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T00:44:35,313 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T00:44:35,313 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T00:44:35,313 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fea72ea5c4b6,46335,1733359474355 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T00:44:35,315 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:44:35,315 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:44:35,315 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:44:35,315 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:44:35,315 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fea72ea5c4b6:0, corePoolSize=10, maxPoolSize=10 2024-12-05T00:44:35,315 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,315 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=2, maxPoolSize=2 2024-12-05T00:44:35,315 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,317 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T00:44:35,317 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T00:44:35,317 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:35,317 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T00:44:35,320 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733359505320 2024-12-05T00:44:35,321 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T00:44:35,321 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T00:44:35,321 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T00:44:35,321 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T00:44:35,321 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T00:44:35,321 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T00:44:35,321 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,322 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T00:44:35,322 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T00:44:35,322 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T00:44:35,328 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T00:44:35,328 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T00:44:35,329 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.large.0-1733359475328,5,FailOnTimeoutGroup] 2024-12-05T00:44:35,329 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(746): ClusterId : 2afb9446-ad7f-49d7-9e08-887edae3137d 2024-12-05T00:44:35,329 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(746): ClusterId : 2afb9446-ad7f-49d7-9e08-887edae3137d 2024-12-05T00:44:35,330 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T00:44:35,330 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T00:44:35,331 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.small.0-1733359475329,5,FailOnTimeoutGroup] 2024-12-05T00:44:35,331 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,331 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T00:44:35,331 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,332 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,342 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T00:44:35,342 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T00:44:35,343 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T00:44:35,343 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T00:44:35,347 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T00:44:35,348 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b 2024-12-05T00:44:35,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:35,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T00:44:35,362 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T00:44:35,362 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:35,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:35,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T00:44:35,369 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T00:44:35,369 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:35,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:35,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T00:44:35,373 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T00:44:35,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:35,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:35,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T00:44:35,381 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T00:44:35,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:35,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:35,382 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T00:44:35,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740 2024-12-05T00:44:35,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740 2024-12-05T00:44:35,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T00:44:35,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T00:44:35,383 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T00:44:35,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T00:44:35,389 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T00:44:35,389 DEBUG [RS:1;fea72ea5c4b6:38585 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c107614, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fea72ea5c4b6/172.17.0.2:0 2024-12-05T00:44:35,390 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:44:35,390 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T00:44:35,390 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69069256, jitterRate=0.029212117195129395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T00:44:35,391 DEBUG [RS:0;fea72ea5c4b6:32821 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f10ad0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fea72ea5c4b6/172.17.0.2:0 2024-12-05T00:44:35,391 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733359475357Initializing all the Stores at 1733359475357Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359475357Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359475358 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359475358Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359475358Cleaning up temporary data from old regions at 1733359475383 (+25 ms)Region opened successfully at 1733359475391 (+8 ms) 2024-12-05T00:44:35,391 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T00:44:35,391 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T00:44:35,391 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T00:44:35,391 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T00:44:35,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T00:44:35,392 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T00:44:35,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733359475391Disabling compacts and flushes for region at 1733359475391Disabling writes for close at 1733359475391Writing region close event to WAL at 1733359475392 (+1 ms)Closed at 1733359475392 2024-12-05T00:44:35,393 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T00:44:35,393 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T00:44:35,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T00:44:35,395 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T00:44:35,396 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T00:44:35,407 DEBUG [RS:0;fea72ea5c4b6:32821 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fea72ea5c4b6:32821 2024-12-05T00:44:35,407 DEBUG [RS:1;fea72ea5c4b6:38585 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fea72ea5c4b6:38585 2024-12-05T00:44:35,408 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T00:44:35,408 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T00:44:35,408 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T00:44:35,408 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T00:44:35,408 DEBUG [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T00:44:35,408 DEBUG [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T00:44:35,409 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(2659): reportForDuty to master=fea72ea5c4b6,46335,1733359474355 with port=32821, startcode=1733359474539 2024-12-05T00:44:35,409 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(2659): reportForDuty to master=fea72ea5c4b6,46335,1733359474355 with port=38585, startcode=1733359474593 2024-12-05T00:44:35,409 DEBUG [RS:1;fea72ea5c4b6:38585 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T00:44:35,409 DEBUG [RS:0;fea72ea5c4b6:32821 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T00:44:35,415 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49351, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T00:44:35,415 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33899, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T00:44:35,416 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46335 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fea72ea5c4b6,32821,1733359474539 2024-12-05T00:44:35,416 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46335 {}] master.ServerManager(517): Registering regionserver=fea72ea5c4b6,32821,1733359474539 2024-12-05T00:44:35,418 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46335 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:35,418 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46335 {}] master.ServerManager(517): Registering regionserver=fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:35,421 DEBUG [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b 2024-12-05T00:44:35,421 DEBUG [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T00:44:35,421 DEBUG [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T00:44:35,423 DEBUG [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b 2024-12-05T00:44:35,423 DEBUG [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T00:44:35,423 DEBUG [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T00:44:35,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T00:44:35,489 DEBUG [RS:1;fea72ea5c4b6:38585 {}] zookeeper.ZKUtil(111): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:35,489 WARN [RS:1;fea72ea5c4b6:38585 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T00:44:35,489 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fea72ea5c4b6,32821,1733359474539] 2024-12-05T00:44:35,489 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fea72ea5c4b6,38585,1733359474593] 2024-12-05T00:44:35,489 INFO [RS:1;fea72ea5c4b6:38585 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:44:35,489 DEBUG [RS:0;fea72ea5c4b6:32821 {}] zookeeper.ZKUtil(111): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fea72ea5c4b6,32821,1733359474539 2024-12-05T00:44:35,490 DEBUG [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:35,490 WARN [RS:0;fea72ea5c4b6:32821 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T00:44:35,490 INFO [RS:0;fea72ea5c4b6:32821 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:44:35,490 DEBUG [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,32821,1733359474539 2024-12-05T00:44:35,494 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T00:44:35,494 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T00:44:35,496 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T00:44:35,496 INFO [RS:0;fea72ea5c4b6:32821 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T00:44:35,496 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,496 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T00:44:35,497 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T00:44:35,497 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,497 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,497 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,497 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,497 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,497 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,497 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=2, maxPoolSize=2 2024-12-05T00:44:35,497 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,497 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,497 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,498 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,498 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,498 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,498 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:44:35,498 DEBUG [RS:0;fea72ea5c4b6:32821 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:44:35,500 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T00:44:35,501 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,501 INFO [RS:1;fea72ea5c4b6:38585 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T00:44:35,501 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,501 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,501 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,501 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,501 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T00:44:35,501 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,501 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,32821,1733359474539-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T00:44:35,501 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T00:44:35,502 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,502 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,502 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,502 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,502 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,502 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,502 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=2, maxPoolSize=2 2024-12-05T00:44:35,502 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,502 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,502 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,502 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,502 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,503 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:35,503 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:44:35,503 DEBUG [RS:1;fea72ea5c4b6:38585 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:44:35,512 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,512 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,512 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,513 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,513 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,513 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,38585,1733359474593-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T00:44:35,519 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T00:44:35,519 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,32821,1733359474539-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,519 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,519 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.Replication(171): fea72ea5c4b6,32821,1733359474539 started 2024-12-05T00:44:35,529 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T00:44:35,529 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,38585,1733359474593-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,530 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,530 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.Replication(171): fea72ea5c4b6,38585,1733359474593 started 2024-12-05T00:44:35,532 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,533 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(1482): Serving as fea72ea5c4b6,32821,1733359474539, RpcServer on fea72ea5c4b6/172.17.0.2:32821, sessionid=0x101a2f2923b0001 2024-12-05T00:44:35,533 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T00:44:35,533 DEBUG [RS:0;fea72ea5c4b6:32821 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fea72ea5c4b6,32821,1733359474539 2024-12-05T00:44:35,533 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,32821,1733359474539' 2024-12-05T00:44:35,533 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T00:44:35,534 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T00:44:35,534 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T00:44:35,534 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T00:44:35,534 DEBUG [RS:0;fea72ea5c4b6:32821 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fea72ea5c4b6,32821,1733359474539 2024-12-05T00:44:35,534 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,32821,1733359474539' 2024-12-05T00:44:35,534 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T00:44:35,535 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T00:44:35,535 DEBUG [RS:0;fea72ea5c4b6:32821 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T00:44:35,536 INFO [RS:0;fea72ea5c4b6:32821 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T00:44:35,536 INFO [RS:0;fea72ea5c4b6:32821 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T00:44:35,545 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:35,545 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(1482): Serving as fea72ea5c4b6,38585,1733359474593, RpcServer on fea72ea5c4b6/172.17.0.2:38585, sessionid=0x101a2f2923b0002 2024-12-05T00:44:35,545 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T00:44:35,545 DEBUG [RS:1;fea72ea5c4b6:38585 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:35,545 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,38585,1733359474593' 2024-12-05T00:44:35,545 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T00:44:35,546 WARN [fea72ea5c4b6:46335 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T00:44:35,546 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T00:44:35,547 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T00:44:35,547 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T00:44:35,547 DEBUG [RS:1;fea72ea5c4b6:38585 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:35,547 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,38585,1733359474593' 2024-12-05T00:44:35,547 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T00:44:35,548 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T00:44:35,548 DEBUG [RS:1;fea72ea5c4b6:38585 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T00:44:35,548 INFO [RS:1;fea72ea5c4b6:38585 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T00:44:35,548 INFO [RS:1;fea72ea5c4b6:38585 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T00:44:35,637 INFO [RS:0;fea72ea5c4b6:32821 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T00:44:35,639 INFO [RS:0;fea72ea5c4b6:32821 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fea72ea5c4b6%2C32821%2C1733359474539, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,32821,1733359474539, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs, maxLogs=55 2024-12-05T00:44:35,649 INFO [RS:1;fea72ea5c4b6:38585 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T00:44:35,650 INFO [RS:1;fea72ea5c4b6:38585 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fea72ea5c4b6%2C38585%2C1733359474593, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,38585,1733359474593, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs, maxLogs=55 2024-12-05T00:44:35,662 INFO [RS:0;fea72ea5c4b6:32821 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,32821,1733359474539/fea72ea5c4b6%2C32821%2C1733359474539.1733359475644 2024-12-05T00:44:35,663 DEBUG [RS:0;fea72ea5c4b6:32821 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T00:44:35,666 INFO [RS:1;fea72ea5c4b6:38585 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,38585,1733359474593/fea72ea5c4b6%2C38585%2C1733359474593.1733359475652 2024-12-05T00:44:35,666 DEBUG [RS:1;fea72ea5c4b6:38585 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T00:44:35,796 DEBUG [fea72ea5c4b6:46335 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-05T00:44:35,797 DEBUG [fea72ea5c4b6:46335 {}] balancer.BalancerClusterState(204): Hosts are {fea72ea5c4b6=0} racks are {/default-rack=0} 2024-12-05T00:44:35,798 DEBUG [fea72ea5c4b6:46335 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T00:44:35,798 DEBUG [fea72ea5c4b6:46335 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T00:44:35,798 DEBUG [fea72ea5c4b6:46335 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T00:44:35,798 DEBUG [fea72ea5c4b6:46335 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T00:44:35,798 INFO [fea72ea5c4b6:46335 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T00:44:35,798 INFO [fea72ea5c4b6:46335 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T00:44:35,798 DEBUG [fea72ea5c4b6:46335 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T00:44:35,798 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:35,800 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fea72ea5c4b6,38585,1733359474593, state=OPENING 2024-12-05T00:44:35,863 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T00:44:35,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:35,894 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:35,894 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T00:44:35,894 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:35,894 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fea72ea5c4b6,38585,1733359474593}] 2024-12-05T00:44:35,894 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:36,047 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T00:44:36,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52929, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T00:44:36,053 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T00:44:36,053 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:44:36,054 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T00:44:36,055 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fea72ea5c4b6%2C38585%2C1733359474593.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,38585,1733359474593, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs, maxLogs=55 2024-12-05T00:44:36,079 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,38585,1733359474593/fea72ea5c4b6%2C38585%2C1733359474593.meta.1733359476056.meta 2024-12-05T00:44:36,079 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T00:44:36,079 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:44:36,079 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T00:44:36,080 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T00:44:36,080 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T00:44:36,080 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T00:44:36,080 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:36,080 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T00:44:36,080 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T00:44:36,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T00:44:36,081 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T00:44:36,081 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:36,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:36,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T00:44:36,082 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T00:44:36,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:36,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:36,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T00:44:36,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T00:44:36,083 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:36,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:36,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T00:44:36,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T00:44:36,083 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:36,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:44:36,084 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T00:44:36,084 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740 2024-12-05T00:44:36,085 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740 2024-12-05T00:44:36,085 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T00:44:36,085 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T00:44:36,085 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T00:44:36,086 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T00:44:36,086 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70172311, jitterRate=0.04564891755580902}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T00:44:36,086 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T00:44:36,087 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733359476080Writing region info on filesystem at 1733359476080Initializing all the Stores at 1733359476080Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359476080Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359476081 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359476081Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359476081Cleaning up temporary data from old regions at 1733359476085 (+4 ms)Running coprocessor post-open hooks at 1733359476086 (+1 ms)Region opened successfully at 1733359476087 (+1 ms) 2024-12-05T00:44:36,087 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733359476047 2024-12-05T00:44:36,090 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T00:44:36,090 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T00:44:36,090 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:36,091 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fea72ea5c4b6,38585,1733359474593, state=OPEN 2024-12-05T00:44:36,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T00:44:36,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T00:44:36,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T00:44:36,130 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:36,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:36,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:36,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:44:36,133 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T00:44:36,133 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fea72ea5c4b6,38585,1733359474593 in 236 msec 2024-12-05T00:44:36,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T00:44:36,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 739 msec 2024-12-05T00:44:36,137 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T00:44:36,137 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T00:44:36,140 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:44:36,141 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,38585,1733359474593, seqNum=-1] 2024-12-05T00:44:36,141 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:44:36,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58699, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:44:36,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 835 msec 2024-12-05T00:44:36,147 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733359476147, completionTime=-1 2024-12-05T00:44:36,147 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-05T00:44:36,147 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T00:44:36,149 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T00:44:36,149 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733359536149 2024-12-05T00:44:36,149 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733359596149 2024-12-05T00:44:36,150 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-05T00:44:36,150 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,46335,1733359474355-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:36,150 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,46335,1733359474355-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:36,150 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,46335,1733359474355-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:36,150 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fea72ea5c4b6:46335, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:36,150 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:36,150 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:36,153 DEBUG [master/fea72ea5c4b6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T00:44:36,155 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.382sec 2024-12-05T00:44:36,155 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T00:44:36,155 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T00:44:36,155 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T00:44:36,155 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T00:44:36,155 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T00:44:36,155 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,46335,1733359474355-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T00:44:36,155 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,46335,1733359474355-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T00:44:36,156 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T00:44:36,156 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T00:44:36,156 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,46335,1733359474355-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:36,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d25b74f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:44:36,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,46335,-1 for getting cluster id 2024-12-05T00:44:36,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:44:36,236 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2afb9446-ad7f-49d7-9e08-887edae3137d' 2024-12-05T00:44:36,238 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:44:36,238 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2afb9446-ad7f-49d7-9e08-887edae3137d" 2024-12-05T00:44:36,239 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c53dee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:44:36,239 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,46335,-1] 2024-12-05T00:44:36,240 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:44:36,240 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:36,242 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:44:36,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5858d96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:44:36,244 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:44:36,246 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,38585,1733359474593, seqNum=-1] 2024-12-05T00:44:36,247 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:44:36,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:44:36,254 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fea72ea5c4b6,46335,1733359474355 2024-12-05T00:44:36,254 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T00:44:36,257 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is fea72ea5c4b6,46335,1733359474355 2024-12-05T00:44:36,257 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@716d4ac2 2024-12-05T00:44:36,257 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T00:44:36,262 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T00:44:36,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46335 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733359476254', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:44:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46335 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733359476254 2024-12-05T00:44:36,265 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733359476254 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T00:44:36,265 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:36,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46335 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733359476254" procId is: 4 2024-12-05T00:44:36,266 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733359476254 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T00:44:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46335 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:44:36,290 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733359476254-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 81e4128c212ccf9ffb399e99be37fd2a, NAME => 'testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733359476254', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b 2024-12-05T00:44:36,302 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733359476254-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:36,302 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733359476254-pool-0 {}] regionserver.HRegion(1722): Closing 81e4128c212ccf9ffb399e99be37fd2a, disabling compactions & flushes 2024-12-05T00:44:36,302 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733359476254-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:36,302 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733359476254-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:36,302 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733359476254-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. after waiting 0 ms 2024-12-05T00:44:36,302 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733359476254-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:36,302 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733359476254-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:36,302 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733359476254-pool-0 {}] regionserver.HRegion(1676): Region close journal for 81e4128c212ccf9ffb399e99be37fd2a: Waiting for close lock at 1733359476302Disabling compacts and flushes for region at 1733359476302Disabling writes for close at 1733359476302Writing region close event to WAL at 1733359476302Closed at 1733359476302 2024-12-05T00:44:36,303 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733359476254 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T00:44:36,304 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733359476303"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733359476303"}]},"ts":"1733359476303"} 2024-12-05T00:44:36,306 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T00:44:36,307 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733359476254 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T00:44:36,308 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733359476254","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359476307"}]},"ts":"1733359476307"} 2024-12-05T00:44:36,310 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733359476254, state=ENABLING in hbase:meta 2024-12-05T00:44:36,310 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {fea72ea5c4b6=0} racks are {/default-rack=0} 2024-12-05T00:44:36,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T00:44:36,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T00:44:36,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T00:44:36,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T00:44:36,311 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T00:44:36,311 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T00:44:36,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T00:44:36,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733359476254, region=81e4128c212ccf9ffb399e99be37fd2a, ASSIGN}] 2024-12-05T00:44:36,312 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733359476254, region=81e4128c212ccf9ffb399e99be37fd2a, ASSIGN 2024-12-05T00:44:36,313 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733359476254, region=81e4128c212ccf9ffb399e99be37fd2a, ASSIGN; state=OFFLINE, location=fea72ea5c4b6,32821,1733359474539; forceNewPlan=false, retain=false 2024-12-05T00:44:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46335 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:44:36,438 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T00:44:36,464 INFO [fea72ea5c4b6:46335 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T00:44:36,464 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=81e4128c212ccf9ffb399e99be37fd2a, regionState=OPENING, regionLocation=fea72ea5c4b6,32821,1733359474539 2024-12-05T00:44:36,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733359476254, region=81e4128c212ccf9ffb399e99be37fd2a, ASSIGN because future has completed 2024-12-05T00:44:36,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81e4128c212ccf9ffb399e99be37fd2a, server=fea72ea5c4b6,32821,1733359474539}] 2024-12-05T00:44:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46335 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:44:36,622 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T00:44:36,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47245, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T00:44:36,625 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:36,625 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 81e4128c212ccf9ffb399e99be37fd2a, NAME => 'testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:44:36,626 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733359476254 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:36,626 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:36,626 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:36,626 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:36,627 INFO [StoreOpener-81e4128c212ccf9ffb399e99be37fd2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:36,629 INFO [StoreOpener-81e4128c212ccf9ffb399e99be37fd2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81e4128c212ccf9ffb399e99be37fd2a columnFamilyName family 2024-12-05T00:44:36,630 DEBUG [StoreOpener-81e4128c212ccf9ffb399e99be37fd2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:36,630 INFO [StoreOpener-81e4128c212ccf9ffb399e99be37fd2a-1 {}] regionserver.HStore(327): Store=81e4128c212ccf9ffb399e99be37fd2a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:44:36,630 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:36,630 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:36,631 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:36,631 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:36,631 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:36,631 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:36,637 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:44:36,637 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 81e4128c212ccf9ffb399e99be37fd2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66977613, jitterRate=-0.0019557923078536987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T00:44:36,637 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:36,638 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 81e4128c212ccf9ffb399e99be37fd2a: Running coprocessor pre-open hook at 1733359476626Writing region info on filesystem at 1733359476626Initializing all the Stores at 1733359476626Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359476626Cleaning up temporary data from old regions at 1733359476631 (+5 ms)Running coprocessor post-open hooks at 1733359476637 (+6 ms)Region opened successfully at 1733359476637 2024-12-05T00:44:36,638 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a., pid=6, masterSystemTime=1733359476621 2024-12-05T00:44:36,641 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:36,641 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:36,641 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=81e4128c212ccf9ffb399e99be37fd2a, regionState=OPEN, openSeqNum=2, regionLocation=fea72ea5c4b6,32821,1733359474539 2024-12-05T00:44:36,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81e4128c212ccf9ffb399e99be37fd2a, server=fea72ea5c4b6,32821,1733359474539 because future has completed 2024-12-05T00:44:36,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T00:44:36,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 81e4128c212ccf9ffb399e99be37fd2a, server=fea72ea5c4b6,32821,1733359474539 in 177 msec 2024-12-05T00:44:36,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T00:44:36,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733359476254, region=81e4128c212ccf9ffb399e99be37fd2a, ASSIGN in 336 msec 2024-12-05T00:44:36,650 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733359476254 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T00:44:36,650 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733359476254","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359476650"}]},"ts":"1733359476650"} 2024-12-05T00:44:36,652 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733359476254, state=ENABLED in hbase:meta 2024-12-05T00:44:36,653 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733359476254 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T00:44:36,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733359476254 in 390 msec 2024-12-05T00:44:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46335 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:44:36,899 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733359476254 get assigned. Timeout = 60000ms 2024-12-05T00:44:36,899 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733359476254 completed 2024-12-05T00:44:36,900 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T00:44:36,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733359476254 assigned to meta. Checking AM states. 2024-12-05T00:44:36,906 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T00:44:36,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733359476254 assigned. 2024-12-05T00:44:36,910 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733359476254', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a., hostname=fea72ea5c4b6,32821,1733359474539, seqNum=2] 2024-12-05T00:44:36,910 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:44:36,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:44:36,916 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 81e4128c212ccf9ffb399e99be37fd2a 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-05T00:44:36,923 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7c8d0cf5856548df98c7aee1bc0df625 2024-12-05T00:44:36,934 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/.tmp/family/7c8d0cf5856548df98c7aee1bc0df625 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-05T00:44:36,935 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/.tmp/family/7c8d0cf5856548df98c7aee1bc0df625 2024-12-05T00:44:36,937 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/.tmp/family/7c8d0cf5856548df98c7aee1bc0df625 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625 2024-12-05T00:44:36,939 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625, delay=998 ms 2024-12-05T00:44:36,940 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T00:44:36,941 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 81e4128c212ccf9ffb399e99be37fd2a in 25ms, sequenceid=6, compaction requested=false 2024-12-05T00:44:36,941 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733359476254' 2024-12-05T00:44:36,942 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 81e4128c212ccf9ffb399e99be37fd2a: 2024-12-05T00:44:37,939 DEBUG [hfile-prefetch-1733359471977 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 7c8d0cf5856548df98c7aee1bc0df625_0. 2024-12-05T00:44:37,940 DEBUG [hfile-prefetch-1733359471977 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 7c8d0cf5856548df98c7aee1bc0df625_117. 2024-12-05T00:44:37,941 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625, with total blocks 2, and data blocks 1 2024-12-05T00:44:37,941 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-05T00:44:37,941 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-05T00:44:37,942 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625. Will sleep 100ms and try the verification again. 2024-12-05T00:44:37,943 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;fea72ea5c4b6:32821,5,FailOnTimeoutGroup] 2024-12-05T00:44:37,943 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fea72ea5c4b6,32821,1733359474539' ***** 2024-12-05T00:44:37,944 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-12-05T00:44:37,946 INFO [RS:0;fea72ea5c4b6:32821 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T00:44:38,003 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T00:44:38,005 INFO [regionserver/fea72ea5c4b6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:38,042 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625, with total blocks 2, and data blocks 1 2024-12-05T00:44:38,042 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-05T00:44:38,042 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-05T00:44:38,042 DEBUG [hfile-prefetch-1733359471977 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625. Will sleep 100ms and try the verification again. 2024-12-05T00:44:38,093 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T00:44:38,093 INFO [RS:0;fea72ea5c4b6:32821 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.persistence 2024-12-05T00:44:38,093 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T00:44:38,093 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T00:44:38,094 INFO [RS:0;fea72ea5c4b6:32821 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T00:44:38,094 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T00:44:38,104 DEBUG [RS:0;fea72ea5c4b6:32821 {}] bucket.BucketCache(1487): Thread RS:0;fea72ea5c4b6:32821 started persisting bucket cache to file 2024-12-05T00:44:38,104 DEBUG [RS:0;fea72ea5c4b6:32821 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T00:44:38,104 DEBUG [RS:0;fea72ea5c4b6:32821 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T00:44:38,111 DEBUG [RS:0;fea72ea5c4b6:32821 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-05T00:44:38,111 DEBUG [RS:0;fea72ea5c4b6:32821 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733359478104 2024-12-05T00:44:38,111 DEBUG [RS:0;fea72ea5c4b6:32821 {}] bucket.BucketCache(1509): Thread RS:0;fea72ea5c4b6:32821 finished persisting bucket cache to file, renaming 2024-12-05T00:44:38,111 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T00:44:38,111 INFO [RS:0;fea72ea5c4b6:32821 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T00:44:38,111 INFO [RS:0;fea72ea5c4b6:32821 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T00:44:38,111 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(3091): Received CLOSE for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,111 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(959): stopping server fea72ea5c4b6,32821,1733359474539 2024-12-05T00:44:38,111 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T00:44:38,111 INFO [RS:0;fea72ea5c4b6:32821 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fea72ea5c4b6:32821. 2024-12-05T00:44:38,111 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T00:44:38,112 DEBUG [RS:0;fea72ea5c4b6:32821 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:38,112 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 81e4128c212ccf9ffb399e99be37fd2a, disabling compactions & flushes 2024-12-05T00:44:38,112 DEBUG [RS:0;fea72ea5c4b6:32821 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:38,112 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:38,112 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:38,112 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T00:44:38,112 DEBUG [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(1325): Online Regions={81e4128c212ccf9ffb399e99be37fd2a=testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a.} 2024-12-05T00:44:38,112 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. after waiting 0 ms 2024-12-05T00:44:38,112 DEBUG [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(1351): Waiting on 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,112 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:38,112 DEBUG [StoreFileCloser-81e4128c212ccf9ffb399e99be37fd2a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625 2024-12-05T00:44:38,112 WARN [hfile-prefetch-1733359471977 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 10 more 2024-12-05T00:44:38,113 DEBUG [hfile-prefetch-1733359471977 {}] hfile.PrefetchExecutor(121): Prefetch completed for 7c8d0cf5856548df98c7aee1bc0df625 2024-12-05T00:44:38,116 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T00:44:38,116 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T00:44:38,116 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T00:44:38,116 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T00:44:38,116 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T00:44:38,116 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:38,116 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 81e4128c212ccf9ffb399e99be37fd2a: Waiting for close lock at 1733359478111Running coprocessor pre-close hooks at 1733359478112 (+1 ms)Disabling compacts and flushes for region at 1733359478112Disabling writes for close at 1733359478112Writing region close event to WAL at 1733359478113 (+1 ms)Running coprocessor post-close hooks at 1733359478116 (+3 ms)Closed at 1733359478116 2024-12-05T00:44:38,117 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:38,118 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T00:44:38,118 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733359478116 2024-12-05T00:44:38,118 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T00:44:38,312 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(976): stopping server fea72ea5c4b6,32821,1733359474539; all regions closed. 2024-12-05T00:44:38,316 DEBUG [RS:0;fea72ea5c4b6:32821 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs 2024-12-05T00:44:38,316 INFO [RS:0;fea72ea5c4b6:32821 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fea72ea5c4b6%2C32821%2C1733359474539:(num 1733359475644) 2024-12-05T00:44:38,316 DEBUG [RS:0;fea72ea5c4b6:32821 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:38,316 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:38,316 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T00:44:38,317 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.ChoreService(370): Chore service for: regionserver/fea72ea5c4b6:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T00:44:38,317 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T00:44:38,317 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T00:44:38,317 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T00:44:38,317 INFO [regionserver/fea72ea5c4b6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T00:44:38,317 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T00:44:38,318 INFO [RS:0;fea72ea5c4b6:32821 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32821 2024-12-05T00:44:38,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fea72ea5c4b6,32821,1733359474539 2024-12-05T00:44:38,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T00:44:38,400 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T00:44:38,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T00:44:38,448 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T00:44:38,455 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T00:44:38,455 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T00:44:38,456 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T00:44:38,456 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T00:44:38,467 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fea72ea5c4b6:0 server-side Connection retries=45 2024-12-05T00:44:38,467 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:38,467 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:38,467 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T00:44:38,467 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:44:38,467 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T00:44:38,467 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T00:44:38,467 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T00:44:38,468 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38365 2024-12-05T00:44:38,469 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38365 connecting to ZooKeeper ensemble=127.0.0.1:50590 2024-12-05T00:44:38,469 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:38,469 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:38,494 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fea72ea5c4b6,32821,1733359474539] 2024-12-05T00:44:38,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:383650x0, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T00:44:38,548 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38365-0x101a2f2923b0003 connected 2024-12-05T00:44:38,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:38,595 INFO [RS:0;fea72ea5c4b6:32821 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T00:44:38,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32821-0x101a2f2923b0001, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:38,595 INFO [RS:0;fea72ea5c4b6:32821 {}] regionserver.HRegionServer(1031): Exiting; stopping=fea72ea5c4b6,32821,1733359474539; zookeeper connection closed. 2024-12-05T00:44:38,595 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@53cbb60f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@53cbb60f 2024-12-05T00:44:38,611 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fea72ea5c4b6,32821,1733359474539 already deleted, retry=false 2024-12-05T00:44:38,611 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38365-0x101a2f2923b0003, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-05T00:44:38,611 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-05T00:44:38,611 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of fea72ea5c4b6,32821,1733359474539 on fea72ea5c4b6,46335,1733359474355 2024-12-05T00:44:38,614 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T00:44:38,615 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.cache 2024-12-05T00:44:38,616 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T00:44:38,616 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T00:44:38,616 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T00:44:38,617 INFO [Thread-199 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T00:44:38,618 INFO [Thread-199 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-05T00:44:38,618 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T00:44:38,619 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T00:44:38,619 INFO [Thread-199 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-05T00:44:38,620 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38365-0x101a2f2923b0003, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T00:44:38,619 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fea72ea5c4b6,32821,1733359474539, splitWal=true, meta=false 2024-12-05T00:44:38,623 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38365-0x101a2f2923b0003, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T00:44:38,623 DEBUG [Thread-199 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-05T00:44:38,623 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for fea72ea5c4b6,32821,1733359474539 (carryingMeta=false) fea72ea5c4b6,32821,1733359474539/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@50c6e387[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-05T00:44:38,623 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure fea72ea5c4b6,32821,1733359474539, splitWal=true, meta=false 2024-12-05T00:44:38,623 INFO [Thread-199 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T00:44:38,623 INFO [Thread-199 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-05T00:44:38,623 INFO [Thread-199 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.persistence is complete. 2024-12-05T00:44:38,625 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): fea72ea5c4b6,32821,1733359474539 had 1 regions 2024-12-05T00:44:38,625 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fea72ea5c4b6,32821,1733359474539, splitWal=true, meta=false, isMeta: false 2024-12-05T00:44:38,626 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,32821,1733359474539-splitting 2024-12-05T00:44:38,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38365 2024-12-05T00:44:38,626 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,32821,1733359474539-splitting dir is empty, no logs to split. 2024-12-05T00:44:38,626 INFO [PEWorker-2 {}] master.SplitWALManager(105): fea72ea5c4b6,32821,1733359474539 WAL count=0, meta=false 2024-12-05T00:44:38,627 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38365 2024-12-05T00:44:38,628 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38365 2024-12-05T00:44:38,628 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,32821,1733359474539-splitting dir is empty, no logs to split. 2024-12-05T00:44:38,628 INFO [PEWorker-2 {}] master.SplitWALManager(105): fea72ea5c4b6,32821,1733359474539 WAL count=0, meta=false 2024-12-05T00:44:38,628 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if fea72ea5c4b6,32821,1733359474539 WAL splitting is done? wals=0, meta=false 2024-12-05T00:44:38,629 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for fea72ea5c4b6,32821,1733359474539 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,32821,1733359474539-splitting does not exist 2024-12-05T00:44:38,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38365 2024-12-05T00:44:38,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38365 2024-12-05T00:44:38,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733359476254, region=81e4128c212ccf9ffb399e99be37fd2a, ASSIGN}] 2024-12-05T00:44:38,632 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer(746): ClusterId : 2afb9446-ad7f-49d7-9e08-887edae3137d 2024-12-05T00:44:38,632 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T00:44:38,632 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733359476254, region=81e4128c212ccf9ffb399e99be37fd2a, ASSIGN 2024-12-05T00:44:38,633 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733359476254, region=81e4128c212ccf9ffb399e99be37fd2a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T00:44:38,652 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T00:44:38,652 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T00:44:38,662 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T00:44:38,663 DEBUG [RS:2;fea72ea5c4b6:38365 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c7e7f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fea72ea5c4b6/172.17.0.2:0 2024-12-05T00:44:38,674 DEBUG [RS:2;fea72ea5c4b6:38365 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;fea72ea5c4b6:38365 2024-12-05T00:44:38,674 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T00:44:38,674 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T00:44:38,674 DEBUG [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T00:44:38,675 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer(2659): reportForDuty to master=fea72ea5c4b6,46335,1733359474355 with port=38365, startcode=1733359478466 2024-12-05T00:44:38,675 DEBUG [RS:2;fea72ea5c4b6:38365 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T00:44:38,677 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42383, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T00:44:38,678 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46335 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fea72ea5c4b6,38365,1733359478466 2024-12-05T00:44:38,678 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46335 {}] master.ServerManager(517): Registering regionserver=fea72ea5c4b6,38365,1733359478466 2024-12-05T00:44:38,679 DEBUG [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b 2024-12-05T00:44:38,679 DEBUG [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T00:44:38,679 DEBUG [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T00:44:38,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T00:44:38,688 DEBUG [RS:2;fea72ea5c4b6:38365 {}] zookeeper.ZKUtil(111): regionserver:38365-0x101a2f2923b0003, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fea72ea5c4b6,38365,1733359478466 2024-12-05T00:44:38,688 WARN [RS:2;fea72ea5c4b6:38365 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T00:44:38,688 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fea72ea5c4b6,38365,1733359478466] 2024-12-05T00:44:38,688 INFO [RS:2;fea72ea5c4b6:38365 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:44:38,688 DEBUG [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,38365,1733359478466 2024-12-05T00:44:38,693 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T00:44:38,696 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T00:44:38,697 INFO [RS:2;fea72ea5c4b6:38365 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T00:44:38,697 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:38,697 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T00:44:38,698 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T00:44:38,698 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:38,698 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:38,698 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:38,698 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:38,698 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:38,699 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:38,699 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=2, maxPoolSize=2 2024-12-05T00:44:38,699 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:38,699 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:38,699 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:38,699 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:38,699 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:38,699 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:44:38,699 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:44:38,699 DEBUG [RS:2;fea72ea5c4b6:38365 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:44:38,700 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:38,700 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:38,700 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:38,700 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:38,700 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:38,700 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,38365,1733359478466-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T00:44:38,713 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T00:44:38,713 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,38365,1733359478466-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:38,713 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:38,713 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.Replication(171): fea72ea5c4b6,38365,1733359478466 started 2024-12-05T00:44:38,731 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:44:38,731 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer(1482): Serving as fea72ea5c4b6,38365,1733359478466, RpcServer on fea72ea5c4b6/172.17.0.2:38365, sessionid=0x101a2f2923b0003 2024-12-05T00:44:38,731 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T00:44:38,731 DEBUG [RS:2;fea72ea5c4b6:38365 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fea72ea5c4b6,38365,1733359478466 2024-12-05T00:44:38,731 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,38365,1733359478466' 2024-12-05T00:44:38,731 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T00:44:38,732 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T00:44:38,733 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T00:44:38,733 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T00:44:38,733 DEBUG [RS:2;fea72ea5c4b6:38365 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fea72ea5c4b6,38365,1733359478466 2024-12-05T00:44:38,733 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,38365,1733359478466' 2024-12-05T00:44:38,733 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T00:44:38,733 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T00:44:38,734 DEBUG [RS:2;fea72ea5c4b6:38365 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T00:44:38,734 INFO [RS:2;fea72ea5c4b6:38365 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T00:44:38,734 INFO [RS:2;fea72ea5c4b6:38365 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T00:44:38,783 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=81e4128c212ccf9ffb399e99be37fd2a, regionState=OPENING, regionLocation=fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:38,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733359476254, region=81e4128c212ccf9ffb399e99be37fd2a, ASSIGN because future has completed 2024-12-05T00:44:38,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81e4128c212ccf9ffb399e99be37fd2a, server=fea72ea5c4b6,38585,1733359474593}] 2024-12-05T00:44:38,834 INFO [RS:2;fea72ea5c4b6:38365 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T00:44:38,835 INFO [RS:2;fea72ea5c4b6:38365 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fea72ea5c4b6%2C38365%2C1733359478466, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,38365,1733359478466, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs, maxLogs=55 2024-12-05T00:44:38,853 INFO [RS:2;fea72ea5c4b6:38365 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/WALs/fea72ea5c4b6,38365,1733359478466/fea72ea5c4b6%2C38365%2C1733359478466.1733359478835 2024-12-05T00:44:38,853 DEBUG [RS:2;fea72ea5c4b6:38365 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T00:44:38,941 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:38,941 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 81e4128c212ccf9ffb399e99be37fd2a, NAME => 'testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:44:38,942 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733359476254 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,942 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:44:38,942 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,942 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,943 INFO [StoreOpener-81e4128c212ccf9ffb399e99be37fd2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,943 INFO [StoreOpener-81e4128c212ccf9ffb399e99be37fd2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81e4128c212ccf9ffb399e99be37fd2a columnFamilyName family 2024-12-05T00:44:38,943 DEBUG [StoreOpener-81e4128c212ccf9ffb399e99be37fd2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:38,945 DEBUG [StoreFileOpener-81e4128c212ccf9ffb399e99be37fd2a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625, delay=994 ms 2024-12-05T00:44:38,947 DEBUG [StoreOpener-81e4128c212ccf9ffb399e99be37fd2a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625 2024-12-05T00:44:38,947 INFO [StoreOpener-81e4128c212ccf9ffb399e99be37fd2a-1 {}] regionserver.HStore(327): Store=81e4128c212ccf9ffb399e99be37fd2a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:44:38,947 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,948 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,948 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,949 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,949 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,950 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,951 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 81e4128c212ccf9ffb399e99be37fd2a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63925117, jitterRate=-0.047441527247428894}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T00:44:38,951 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:38,952 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 81e4128c212ccf9ffb399e99be37fd2a: Running coprocessor pre-open hook at 1733359478942Writing region info on filesystem at 1733359478942Initializing all the Stores at 1733359478942Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359478942Cleaning up temporary data from old regions at 1733359478949 (+7 ms)Running coprocessor post-open hooks at 1733359478951 (+2 ms)Region opened successfully at 1733359478951 2024-12-05T00:44:38,952 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a., pid=9, masterSystemTime=1733359478939 2024-12-05T00:44:38,955 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:38,955 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:38,955 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=81e4128c212ccf9ffb399e99be37fd2a, regionState=OPEN, openSeqNum=10, regionLocation=fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:38,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81e4128c212ccf9ffb399e99be37fd2a, server=fea72ea5c4b6,38585,1733359474593 because future has completed 2024-12-05T00:44:38,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-05T00:44:38,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 81e4128c212ccf9ffb399e99be37fd2a, server=fea72ea5c4b6,38585,1733359474593 in 172 msec 2024-12-05T00:44:38,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T00:44:38,963 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server fea72ea5c4b6,32821,1733359474539 after splitting done 2024-12-05T00:44:38,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733359476254, region=81e4128c212ccf9ffb399e99be37fd2a, ASSIGN in 330 msec 2024-12-05T00:44:38,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure fea72ea5c4b6,32821,1733359474539, splitWal=true, meta=false in 349 msec 2024-12-05T00:44:39,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T00:44:39,231 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:44:39,232 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:39,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:39,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:39,232 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T00:44:39,232 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1731608923, stopped=false 2024-12-05T00:44:39,233 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fea72ea5c4b6,46335,1733359474355 2024-12-05T00:44:39,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:44:39,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:39,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:39,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38365-0x101a2f2923b0003, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:39,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:39,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:39,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38365-0x101a2f2923b0003, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:39,246 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T00:44:39,246 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:44:39,246 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:39,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:39,246 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:39,246 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fea72ea5c4b6,38585,1733359474593' ***** 2024-12-05T00:44:39,246 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38365-0x101a2f2923b0003, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:39,246 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T00:44:39,246 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fea72ea5c4b6,38365,1733359478466' ***** 2024-12-05T00:44:39,247 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T00:44:39,247 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:39,247 INFO [RS:2;fea72ea5c4b6:38365 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T00:44:39,247 INFO [RS:1;fea72ea5c4b6:38585 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T00:44:39,247 INFO [RS:2;fea72ea5c4b6:38365 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.persistence 2024-12-05T00:44:39,247 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T00:44:39,247 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T00:44:39,247 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T00:44:39,247 INFO [RS:2;fea72ea5c4b6:38365 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T00:44:39,247 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T00:44:39,247 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T00:44:39,247 INFO [RS:1;fea72ea5c4b6:38585 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7646d524-d3fc-649e-e50b-7b734edd3a79/bucket.persistence 2024-12-05T00:44:39,247 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T00:44:39,247 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T00:44:39,247 INFO [RS:1;fea72ea5c4b6:38585 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T00:44:39,247 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T00:44:39,257 DEBUG [RS:2;fea72ea5c4b6:38365 {}] bucket.BucketCache(1487): Thread RS:2;fea72ea5c4b6:38365 started persisting bucket cache to file 2024-12-05T00:44:39,257 DEBUG [RS:1;fea72ea5c4b6:38585 {}] bucket.BucketCache(1487): Thread RS:1;fea72ea5c4b6:38585 started persisting bucket cache to file 2024-12-05T00:44:39,257 DEBUG [RS:2;fea72ea5c4b6:38365 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T00:44:39,258 DEBUG [RS:2;fea72ea5c4b6:38365 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T00:44:39,258 DEBUG [RS:1;fea72ea5c4b6:38585 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T00:44:39,258 DEBUG [RS:1;fea72ea5c4b6:38585 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T00:44:39,261 DEBUG [RS:2;fea72ea5c4b6:38365 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T00:44:39,261 DEBUG [RS:2;fea72ea5c4b6:38365 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733359479257 2024-12-05T00:44:39,261 DEBUG [RS:2;fea72ea5c4b6:38365 {}] bucket.BucketCache(1509): Thread RS:2;fea72ea5c4b6:38365 finished persisting bucket cache to file, renaming 2024-12-05T00:44:39,262 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T00:44:39,262 INFO [RS:2;fea72ea5c4b6:38365 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T00:44:39,262 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T00:44:39,262 INFO [RS:2;fea72ea5c4b6:38365 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T00:44:39,262 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer(959): stopping server fea72ea5c4b6,38365,1733359478466 2024-12-05T00:44:39,262 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T00:44:39,262 WARN [Thread-209 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-12-05T00:44:39,262 INFO [RS:2;fea72ea5c4b6:38365 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;fea72ea5c4b6:38365. 2024-12-05T00:44:39,262 DEBUG [RS:2;fea72ea5c4b6:38365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:39,262 DEBUG [RS:2;fea72ea5c4b6:38365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:39,262 ERROR [RS:1;fea72ea5c4b6:38585 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:44:39,262 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer(976): stopping server fea72ea5c4b6,38365,1733359478466; all regions closed. 2024-12-05T00:44:39,262 DEBUG [RS:1;fea72ea5c4b6:38585 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T00:44:39,262 DEBUG [RS:1;fea72ea5c4b6:38585 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733359479257 2024-12-05T00:44:39,262 DEBUG [RS:1;fea72ea5c4b6:38585 {}] bucket.BucketCache(1509): Thread RS:1;fea72ea5c4b6:38585 finished persisting bucket cache to file, renaming 2024-12-05T00:44:39,263 WARN [RS:1;fea72ea5c4b6:38585 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-05T00:44:39,263 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T00:44:39,263 INFO [RS:1;fea72ea5c4b6:38585 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T00:44:39,263 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T00:44:39,263 INFO [RS:1;fea72ea5c4b6:38585 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T00:44:39,263 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(3091): Received CLOSE for 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:39,264 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(959): stopping server fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:39,264 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T00:44:39,264 INFO [RS:1;fea72ea5c4b6:38585 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fea72ea5c4b6:38585. 2024-12-05T00:44:39,264 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T00:44:39,264 DEBUG [RS:1;fea72ea5c4b6:38585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:39,264 DEBUG [RS:1;fea72ea5c4b6:38585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:39,264 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 81e4128c212ccf9ffb399e99be37fd2a, disabling compactions & flushes 2024-12-05T00:44:39,264 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:39,264 DEBUG [RS:2;fea72ea5c4b6:38365 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs 2024-12-05T00:44:39,264 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T00:44:39,264 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T00:44:39,264 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:39,264 INFO [RS:2;fea72ea5c4b6:38365 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fea72ea5c4b6%2C38365%2C1733359478466:(num 1733359478835) 2024-12-05T00:44:39,264 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T00:44:39,264 DEBUG [RS:2;fea72ea5c4b6:38365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:39,264 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. after waiting 0 ms 2024-12-05T00:44:39,264 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:39,265 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T00:44:39,265 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:39,265 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T00:44:39,265 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-05T00:44:39,265 DEBUG [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 81e4128c212ccf9ffb399e99be37fd2a=testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a.} 2024-12-05T00:44:39,265 DEBUG [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 81e4128c212ccf9ffb399e99be37fd2a 2024-12-05T00:44:39,265 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T00:44:39,265 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.ChoreService(370): Chore service for: regionserver/fea72ea5c4b6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T00:44:39,265 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T00:44:39,265 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T00:44:39,265 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T00:44:39,265 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T00:44:39,265 INFO [regionserver/fea72ea5c4b6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T00:44:39,265 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T00:44:39,265 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T00:44:39,265 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T00:44:39,265 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T00:44:39,265 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T00:44:39,265 INFO [RS:2;fea72ea5c4b6:38365 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38365 2024-12-05T00:44:39,265 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-12-05T00:44:39,272 DEBUG [StoreFileCloser-81e4128c212ccf9ffb399e99be37fd2a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/family/7c8d0cf5856548df98c7aee1bc0df625 2024-12-05T00:44:39,281 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/default/testBlockEvictionOnGracefulStop1733359476254/81e4128c212ccf9ffb399e99be37fd2a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-05T00:44:39,281 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 20fb9fe95de3402f93b4874620722cc9 2024-12-05T00:44:39,281 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:39,281 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 81e4128c212ccf9ffb399e99be37fd2a: Waiting for close lock at 1733359479264Running coprocessor pre-close hooks at 1733359479264Disabling compacts and flushes for region at 1733359479264Disabling writes for close at 1733359479265 (+1 ms)Writing region close event to WAL at 1733359479272 (+7 ms)Running coprocessor post-close hooks at 1733359479281 (+9 ms)Closed at 1733359479281 2024-12-05T00:44:39,282 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a. 2024-12-05T00:44:39,293 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/info/20fb9fe95de3402f93b4874620722cc9 is 209, key is testBlockEvictionOnGracefulStop1733359476254,,1733359476262.81e4128c212ccf9ffb399e99be37fd2a./info:regioninfo/1733359478955/Put/seqid=0 2024-12-05T00:44:39,293 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/info/20fb9fe95de3402f93b4874620722cc9 2024-12-05T00:44:39,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38365-0x101a2f2923b0003, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fea72ea5c4b6,38365,1733359478466 2024-12-05T00:44:39,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T00:44:39,298 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T00:44:39,298 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007f8a30477b20@73eec40f rejected from java.util.concurrent.ThreadPoolExecutor@36ec4416[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 3] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-05T00:44:39,299 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fea72ea5c4b6,38365,1733359478466] 2024-12-05T00:44:39,301 INFO [regionserver/fea72ea5c4b6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:39,302 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e97176a25cf94076b45dd1cd000f3e1a 2024-12-05T00:44:39,316 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/ns/e97176a25cf94076b45dd1cd000f3e1a is 43, key is default/ns:d/1733359476143/Put/seqid=0 2024-12-05T00:44:39,317 INFO [regionserver/fea72ea5c4b6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:39,317 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/ns/e97176a25cf94076b45dd1cd000f3e1a 2024-12-05T00:44:39,319 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fea72ea5c4b6,38365,1733359478466 already deleted, retry=false 2024-12-05T00:44:39,319 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fea72ea5c4b6,38365,1733359478466 expired; onlineServers=1 2024-12-05T00:44:39,327 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 142647af3ab2422db547dd139616640f 2024-12-05T00:44:39,341 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/table/142647af3ab2422db547dd139616640f is 80, key is testBlockEvictionOnGracefulStop1733359476254/table:state/1733359476650/Put/seqid=0 2024-12-05T00:44:39,341 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/table/142647af3ab2422db547dd139616640f 2024-12-05T00:44:39,344 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/info/20fb9fe95de3402f93b4874620722cc9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/info/20fb9fe95de3402f93b4874620722cc9 2024-12-05T00:44:39,345 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/info/20fb9fe95de3402f93b4874620722cc9, delay=958 ms 2024-12-05T00:44:39,346 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/info/20fb9fe95de3402f93b4874620722cc9, entries=14, sequenceid=13, filesize=7.5 K 2024-12-05T00:44:39,346 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/ns/e97176a25cf94076b45dd1cd000f3e1a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/ns/e97176a25cf94076b45dd1cd000f3e1a 2024-12-05T00:44:39,347 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/ns/e97176a25cf94076b45dd1cd000f3e1a, delay=935 ms 2024-12-05T00:44:39,348 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/ns/e97176a25cf94076b45dd1cd000f3e1a, entries=2, sequenceid=13, filesize=5.0 K 2024-12-05T00:44:39,348 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/.tmp/table/142647af3ab2422db547dd139616640f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/table/142647af3ab2422db547dd139616640f 2024-12-05T00:44:39,349 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/table/142647af3ab2422db547dd139616640f, delay=973 ms 2024-12-05T00:44:39,350 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/table/142647af3ab2422db547dd139616640f, entries=2, sequenceid=13, filesize=5.3 K 2024-12-05T00:44:39,351 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=13, compaction requested=false 2024-12-05T00:44:39,351 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/info/20fb9fe95de3402f93b4874620722cc9 2024-12-05T00:44:39,352 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/ns/e97176a25cf94076b45dd1cd000f3e1a 2024-12-05T00:44:39,352 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/table/142647af3ab2422db547dd139616640f 2024-12-05T00:44:39,356 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-12-05T00:44:39,356 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T00:44:39,356 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T00:44:39,357 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733359479265Running coprocessor pre-close hooks at 1733359479265Disabling compacts and flushes for region at 1733359479265Disabling writes for close at 1733359479265Obtaining lock to block concurrent updates at 1733359479265Preparing flush snapshotting stores in 1588230740 at 1733359479265Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1733359479266 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733359479266Flushing 1588230740/info: creating writer at 1733359479266Flushing 1588230740/info: appending metadata at 1733359479292 (+26 ms)Flushing 1588230740/info: closing flushed file at 1733359479292Flushing 1588230740/ns: creating writer at 1733359479295 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733359479316 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733359479316Flushing 1588230740/table: creating writer at 1733359479320 (+4 ms)Flushing 1588230740/table: appending metadata at 1733359479340 (+20 ms)Flushing 1588230740/table: closing flushed file at 1733359479340Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4da259ed: reopening flushed file at 1733359479344 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e0f1faf: reopening flushed file at 1733359479346 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74abf7c0: reopening flushed file at 1733359479348 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=13, compaction requested=false at 1733359479351 (+3 ms)Writing region close event to WAL at 1733359479352 (+1 ms)Running coprocessor post-close hooks at 1733359479356 (+4 ms)Closed at 1733359479356 2024-12-05T00:44:39,357 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T00:44:39,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38365-0x101a2f2923b0003, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:39,409 INFO [RS:2;fea72ea5c4b6:38365 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T00:44:39,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38365-0x101a2f2923b0003, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:39,409 INFO [RS:2;fea72ea5c4b6:38365 {}] regionserver.HRegionServer(1031): Exiting; stopping=fea72ea5c4b6,38365,1733359478466; zookeeper connection closed. 2024-12-05T00:44:39,409 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@447c1286 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@447c1286 2024-12-05T00:44:39,465 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(976): stopping server fea72ea5c4b6,38585,1733359474593; all regions closed. 2024-12-05T00:44:39,467 DEBUG [RS:1;fea72ea5c4b6:38585 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs 2024-12-05T00:44:39,467 INFO [RS:1;fea72ea5c4b6:38585 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fea72ea5c4b6%2C38585%2C1733359474593.meta:.meta(num 1733359476056) 2024-12-05T00:44:39,468 DEBUG [RS:1;fea72ea5c4b6:38585 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/oldWALs 2024-12-05T00:44:39,468 INFO [RS:1;fea72ea5c4b6:38585 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fea72ea5c4b6%2C38585%2C1733359474593:(num 1733359475652) 2024-12-05T00:44:39,468 DEBUG [RS:1;fea72ea5c4b6:38585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:39,468 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:39,468 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T00:44:39,469 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.ChoreService(370): Chore service for: regionserver/fea72ea5c4b6:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T00:44:39,469 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T00:44:39,469 INFO [regionserver/fea72ea5c4b6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T00:44:39,469 INFO [RS:1;fea72ea5c4b6:38585 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38585 2024-12-05T00:44:39,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T00:44:39,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fea72ea5c4b6,38585,1733359474593 2024-12-05T00:44:39,483 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T00:44:39,493 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fea72ea5c4b6,38585,1733359474593] 2024-12-05T00:44:39,504 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fea72ea5c4b6,38585,1733359474593 already deleted, retry=false 2024-12-05T00:44:39,504 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fea72ea5c4b6,38585,1733359474593 expired; onlineServers=0 2024-12-05T00:44:39,504 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fea72ea5c4b6,46335,1733359474355' ***** 2024-12-05T00:44:39,504 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T00:44:39,504 INFO [M:0;fea72ea5c4b6:46335 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T00:44:39,504 INFO [M:0;fea72ea5c4b6:46335 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T00:44:39,504 DEBUG [M:0;fea72ea5c4b6:46335 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T00:44:39,504 DEBUG [M:0;fea72ea5c4b6:46335 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T00:44:39,504 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T00:44:39,504 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.small.0-1733359475329 {}] cleaner.HFileCleaner(306): Exit Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.small.0-1733359475329,5,FailOnTimeoutGroup] 2024-12-05T00:44:39,505 INFO [M:0;fea72ea5c4b6:46335 {}] hbase.ChoreService(370): Chore service for: master/fea72ea5c4b6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T00:44:39,505 INFO [M:0;fea72ea5c4b6:46335 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T00:44:39,505 DEBUG [M:0;fea72ea5c4b6:46335 {}] master.HMaster(1795): Stopping service threads 2024-12-05T00:44:39,505 INFO [M:0;fea72ea5c4b6:46335 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T00:44:39,505 INFO [M:0;fea72ea5c4b6:46335 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T00:44:39,505 INFO [M:0;fea72ea5c4b6:46335 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T00:44:39,505 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T00:44:39,508 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.large.0-1733359475328 {}] cleaner.HFileCleaner(306): Exit Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.large.0-1733359475328,5,FailOnTimeoutGroup] 2024-12-05T00:44:39,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T00:44:39,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:39,568 DEBUG [M:0;fea72ea5c4b6:46335 {}] zookeeper.ZKUtil(347): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T00:44:39,569 WARN [M:0;fea72ea5c4b6:46335 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T00:44:39,569 INFO [M:0;fea72ea5c4b6:46335 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/.lastflushedseqids 2024-12-05T00:44:39,577 INFO [M:0;fea72ea5c4b6:46335 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T00:44:39,578 INFO [M:0;fea72ea5c4b6:46335 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T00:44:39,578 DEBUG [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T00:44:39,578 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:39,578 DEBUG [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:39,578 DEBUG [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T00:44:39,578 DEBUG [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:39,578 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-12-05T00:44:39,586 DEBUG [M:0;fea72ea5c4b6:46335 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 75b42bedd70a45d5830f45869be7a012 2024-12-05T00:44:39,593 INFO [RS:1;fea72ea5c4b6:38585 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T00:44:39,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:39,593 INFO [RS:1;fea72ea5c4b6:38585 {}] regionserver.HRegionServer(1031): Exiting; stopping=fea72ea5c4b6,38585,1733359474593; zookeeper connection closed. 2024-12-05T00:44:39,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38585-0x101a2f2923b0002, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:39,594 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7717e3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7717e3 2024-12-05T00:44:39,594 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-05T00:44:39,597 DEBUG [M:0;fea72ea5c4b6:46335 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/75b42bedd70a45d5830f45869be7a012 is 82, key is hbase:meta,,1/info:regioninfo/1733359476090/Put/seqid=0 2024-12-05T00:44:39,598 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/75b42bedd70a45d5830f45869be7a012 2024-12-05T00:44:39,604 DEBUG [M:0;fea72ea5c4b6:46335 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7562d507a8f147c790d2bb1afaf2438c 2024-12-05T00:44:39,620 DEBUG [M:0;fea72ea5c4b6:46335 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7562d507a8f147c790d2bb1afaf2438c is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733359476654/Put/seqid=0 2024-12-05T00:44:39,620 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7562d507a8f147c790d2bb1afaf2438c 2024-12-05T00:44:39,645 DEBUG [M:0;fea72ea5c4b6:46335 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ec57a1b139f24247866d3b2b2ac7ad92 2024-12-05T00:44:39,666 DEBUG [M:0;fea72ea5c4b6:46335 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec57a1b139f24247866d3b2b2ac7ad92 is 69, key is fea72ea5c4b6,38365,1733359478466/rs:state/1733359478678/Put/seqid=0 2024-12-05T00:44:39,667 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec57a1b139f24247866d3b2b2ac7ad92 2024-12-05T00:44:39,669 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ec57a1b139f24247866d3b2b2ac7ad92 2024-12-05T00:44:39,670 DEBUG [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/75b42bedd70a45d5830f45869be7a012 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/75b42bedd70a45d5830f45869be7a012 2024-12-05T00:44:39,673 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/75b42bedd70a45d5830f45869be7a012, entries=8, sequenceid=83, filesize=5.5 K 2024-12-05T00:44:39,673 DEBUG [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7562d507a8f147c790d2bb1afaf2438c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7562d507a8f147c790d2bb1afaf2438c 2024-12-05T00:44:39,675 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7562d507a8f147c790d2bb1afaf2438c, entries=9, sequenceid=83, filesize=6.4 K 2024-12-05T00:44:39,675 DEBUG [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec57a1b139f24247866d3b2b2ac7ad92 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ec57a1b139f24247866d3b2b2ac7ad92 2024-12-05T00:44:39,677 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ec57a1b139f24247866d3b2b2ac7ad92 2024-12-05T00:44:39,677 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/53e23311-a4e3-0b58-35d1-1ae32abe67bd/07a78aa2-e1e2-1284-edbb-cbc038d2181b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ec57a1b139f24247866d3b2b2ac7ad92, entries=3, sequenceid=83, filesize=5.3 K 2024-12-05T00:44:39,678 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=83, compaction requested=false 2024-12-05T00:44:39,681 INFO [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:39,682 DEBUG [M:0;fea72ea5c4b6:46335 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733359479578Disabling compacts and flushes for region at 1733359479578Disabling writes for close at 1733359479578Obtaining lock to block concurrent updates at 1733359479578Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733359479578Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1733359479578Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733359479579 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733359479579Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733359479597 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733359479597Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733359479599 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733359479619 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733359479619Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733359479622 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733359479665 (+43 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733359479665Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c2477dd: reopening flushed file at 1733359479669 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1160d9d: reopening flushed file at 1733359479673 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52d37937: reopening flushed file at 1733359479675 (+2 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=83, compaction requested=false at 1733359479678 (+3 ms)Writing region close event to WAL at 1733359479681 (+3 ms)Closed at 1733359479681 2024-12-05T00:44:39,682 INFO [M:0;fea72ea5c4b6:46335 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T00:44:39,683 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T00:44:39,683 INFO [M:0;fea72ea5c4b6:46335 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46335 2024-12-05T00:44:39,683 INFO [M:0;fea72ea5c4b6:46335 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T00:44:39,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:39,793 INFO [M:0;fea72ea5c4b6:46335 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T00:44:39,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46335-0x101a2f2923b0000, quorum=127.0.0.1:50590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:39,795 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T00:44:39,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-05T00:44:39,797 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:44:39,803 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=353 (was 310) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10061 (was 10192)