2024-11-23 03:45:09,162 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-23 03:45:09,202 main DEBUG Took 0.037380 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 03:45:09,203 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 03:45:09,203 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 03:45:09,205 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 03:45:09,207 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,216 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 03:45:09,233 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,235 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,236 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,237 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,237 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,238 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,239 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,239 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,240 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,240 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,241 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,241 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,242 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,243 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,243 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,244 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,244 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,244 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,245 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,245 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,246 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,246 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,247 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,248 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 03:45:09,248 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,249 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 03:45:09,250 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 03:45:09,252 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 03:45:09,254 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 03:45:09,255 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 03:45:09,257 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 03:45:09,257 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 03:45:09,267 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 03:45:09,270 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 03:45:09,272 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 03:45:09,272 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 03:45:09,273 main DEBUG createAppenders(={Console}) 2024-11-23 03:45:09,274 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-23 03:45:09,274 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-23 03:45:09,274 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-23 03:45:09,275 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 03:45:09,276 main DEBUG OutputStream closed 2024-11-23 03:45:09,276 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 03:45:09,276 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 03:45:09,277 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-23 03:45:09,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 03:45:09,364 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 03:45:09,365 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 03:45:09,366 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 03:45:09,366 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 03:45:09,366 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 03:45:09,367 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 03:45:09,367 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 03:45:09,368 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 03:45:09,368 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 03:45:09,368 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 03:45:09,369 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 03:45:09,369 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 03:45:09,369 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 03:45:09,370 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 03:45:09,370 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 03:45:09,370 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 03:45:09,371 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 03:45:09,373 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 03:45:09,373 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-23 03:45:09,374 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 03:45:09,375 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-23T03:45:09,388 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-23 03:45:09,391 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 03:45:09,391 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T03:45:09,685 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054 2024-11-23T03:45:09,715 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=368, ProcessCount=11, AvailableMemoryMB=7523 2024-11-23T03:45:09,817 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T03:45:09,908 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:09,927 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/cluster_7c34b210-71e4-4d3b-4e38-93aa83efdfdb, deleteOnExit=true 2024-11-23T03:45:10,021 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/cluster_7c34b210-71e4-4d3b-4e38-93aa83efdfdb/zookeeper_0, clientPort=60839, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/cluster_7c34b210-71e4-4d3b-4e38-93aa83efdfdb/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/cluster_7c34b210-71e4-4d3b-4e38-93aa83efdfdb/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T03:45:10,037 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60839 2024-11-23T03:45:10,037 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:10,122 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd with version=8 2024-11-23T03:45:10,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/hbase-staging 2024-11-23T03:45:10,228 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T03:45:10,547 INFO [Time-limited test {}] client.ConnectionUtils(128): master/237a1da887a0:0 server-side Connection retries=45 2024-11-23T03:45:10,562 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:10,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:10,574 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T03:45:10,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:10,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T03:45:10,730 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T03:45:10,839 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T03:45:10,887 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T03:45:10,900 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T03:45:10,905 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T03:45:10,956 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17973 (auto-detected) 2024-11-23T03:45:10,957 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T03:45:10,986 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32933 2024-11-23T03:45:11,014 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32933 connecting to ZooKeeper ensemble=127.0.0.1:60839 2024-11-23T03:45:11,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:329330x0, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T03:45:11,058 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32933-0x10142ed88000000 connected 2024-11-23T03:45:11,090 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:11,091 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:11,099 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:11,103 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd, hbase.cluster.distributed=false 2024-11-23T03:45:11,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T03:45:11,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32933 2024-11-23T03:45:11,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32933 2024-11-23T03:45:11,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32933 2024-11-23T03:45:11,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32933 2024-11-23T03:45:11,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32933 2024-11-23T03:45:11,229 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/237a1da887a0:0 server-side Connection retries=45 2024-11-23T03:45:11,231 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:11,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:11,232 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T03:45:11,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:11,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T03:45:11,235 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T03:45:11,238 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T03:45:11,239 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39613 2024-11-23T03:45:11,241 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39613 connecting to ZooKeeper ensemble=127.0.0.1:60839 2024-11-23T03:45:11,241 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:11,242 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:11,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396130x0, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T03:45:11,247 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:396130x0, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:11,247 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39613-0x10142ed88000001 connected 2024-11-23T03:45:11,253 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T03:45:11,266 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/bucket.cache 2024-11-23T03:45:11,272 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T03:45:11,273 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T03:45:11,275 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T03:45:11,277 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T03:45:11,277 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T03:45:11,284 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T03:45:11,288 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T03:45:11,289 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/bucket.persistence is complete. 2024-11-23T03:45:11,291 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T03:45:11,293 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T03:45:11,299 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T03:45:11,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39613 2024-11-23T03:45:11,300 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39613 2024-11-23T03:45:11,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39613 2024-11-23T03:45:11,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39613 2024-11-23T03:45:11,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39613 2024-11-23T03:45:11,324 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/237a1da887a0:0 server-side Connection retries=45 2024-11-23T03:45:11,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:11,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:11,324 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T03:45:11,325 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:11,325 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T03:45:11,325 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T03:45:11,325 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T03:45:11,326 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44315 2024-11-23T03:45:11,328 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44315 connecting to ZooKeeper ensemble=127.0.0.1:60839 2024-11-23T03:45:11,329 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:11,329 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:11,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:443150x0, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T03:45:11,335 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:443150x0, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:11,335 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44315-0x10142ed88000002 connected 2024-11-23T03:45:11,335 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T03:45:11,336 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/bucket.cache 2024-11-23T03:45:11,336 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T03:45:11,337 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T03:45:11,337 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T03:45:11,339 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T03:45:11,339 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T03:45:11,339 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T03:45:11,340 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T03:45:11,340 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T03:45:11,340 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/bucket.persistence is complete. 2024-11-23T03:45:11,341 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T03:45:11,343 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T03:45:11,348 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44315 2024-11-23T03:45:11,349 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44315 2024-11-23T03:45:11,349 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44315 2024-11-23T03:45:11,350 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44315 2024-11-23T03:45:11,350 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44315 2024-11-23T03:45:11,367 DEBUG [M:0;237a1da887a0:32933 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;237a1da887a0:32933 2024-11-23T03:45:11,369 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/237a1da887a0,32933,1732333510314 2024-11-23T03:45:11,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:11,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:11,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:11,379 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/237a1da887a0,32933,1732333510314 2024-11-23T03:45:11,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T03:45:11,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:11,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T03:45:11,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:11,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:11,415 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T03:45:11,416 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/237a1da887a0,32933,1732333510314 from backup master directory 2024-11-23T03:45:11,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:11,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:11,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/237a1da887a0,32933,1732333510314 2024-11-23T03:45:11,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:11,420 WARN [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T03:45:11,421 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=237a1da887a0,32933,1732333510314 2024-11-23T03:45:11,423 INFO [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T03:45:11,425 INFO [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T03:45:11,433 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/hbase.id] with ID: b7834cd3-746d-48f8-962a-06940d656df8 2024-11-23T03:45:11,433 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/.tmp/hbase.id 2024-11-23T03:45:11,439 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/hbase.id] 2024-11-23T03:45:11,481 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:11,487 INFO [master/237a1da887a0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T03:45:11,506 INFO [master/237a1da887a0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-23T03:45:11,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:11,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:11,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:11,539 INFO [master/237a1da887a0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T03:45:11,542 INFO [master/237a1da887a0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T03:45:11,592 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T03:45:11,599 INFO [master/237a1da887a0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T03:45:11,661 INFO [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store 2024-11-23T03:45:11,685 INFO [master/237a1da887a0:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T03:45:11,688 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:11,690 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T03:45:11,690 INFO [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:11,690 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:11,692 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T03:45:11,692 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:11,692 INFO [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:11,693 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732333511690Disabling compacts and flushes for region at 1732333511690Disabling writes for close at 1732333511692 (+2 ms)Writing region close event to WAL at 1732333511692Closed at 1732333511692 2024-11-23T03:45:11,693 WARN [master/237a1da887a0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/.initializing 2024-11-23T03:45:11,693 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/WALs/237a1da887a0,32933,1732333510314 2024-11-23T03:45:11,708 INFO [master/237a1da887a0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T03:45:11,724 INFO [master/237a1da887a0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=237a1da887a0%2C32933%2C1732333510314, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/WALs/237a1da887a0,32933,1732333510314, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/oldWALs, maxLogs=10 2024-11-23T03:45:11,767 INFO [master/237a1da887a0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/WALs/237a1da887a0,32933,1732333510314/237a1da887a0%2C32933%2C1732333510314.1732333511731 2024-11-23T03:45:11,767 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T03:45:11,768 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T03:45:11,769 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:11,772 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,773 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,842 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T03:45:11,847 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:11,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:11,849 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,855 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T03:45:11,855 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:11,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T03:45:11,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T03:45:11,861 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:11,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T03:45:11,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T03:45:11,867 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:11,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T03:45:11,868 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,871 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,872 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,875 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,875 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,878 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T03:45:11,880 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:11,888 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T03:45:11,888 INFO [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60536374, jitterRate=-0.09793773293495178}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T03:45:11,895 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732333511785Initializing all the Stores at 1732333511785Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333511786 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333511788 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333511789 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333511789Cleaning up temporary data from old regions at 1732333511875 (+86 ms)Region opened successfully at 1732333511894 (+19 ms) 2024-11-23T03:45:11,896 INFO [master/237a1da887a0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T03:45:11,934 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604b4acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=237a1da887a0/172.17.0.2:0 2024-11-23T03:45:11,970 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T03:45:11,982 INFO [master/237a1da887a0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T03:45:11,982 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T03:45:11,986 INFO [master/237a1da887a0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T03:45:11,986 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T03:45:11,992 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-23T03:45:11,993 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T03:45:12,018 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T03:45:12,027 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T03:45:12,030 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T03:45:12,032 INFO [master/237a1da887a0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T03:45:12,033 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T03:45:12,037 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T03:45:12,039 INFO [master/237a1da887a0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T03:45:12,042 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T03:45:12,044 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T03:45:12,046 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T03:45:12,047 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T03:45:12,066 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T03:45:12,068 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T03:45:12,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:12,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:12,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:12,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,085 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=237a1da887a0,32933,1732333510314, sessionid=0x10142ed88000000, setting cluster-up flag (Was=false) 2024-11-23T03:45:12,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,107 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T03:45:12,109 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=237a1da887a0,32933,1732333510314 2024-11-23T03:45:12,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,120 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T03:45:12,122 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=237a1da887a0,32933,1732333510314 2024-11-23T03:45:12,128 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T03:45:12,154 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(746): ClusterId : b7834cd3-746d-48f8-962a-06940d656df8 2024-11-23T03:45:12,155 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(746): ClusterId : b7834cd3-746d-48f8-962a-06940d656df8 2024-11-23T03:45:12,157 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T03:45:12,157 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T03:45:12,162 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T03:45:12,162 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T03:45:12,162 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T03:45:12,163 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T03:45:12,166 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T03:45:12,167 DEBUG [RS:1;237a1da887a0:44315 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@334940ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=237a1da887a0/172.17.0.2:0 2024-11-23T03:45:12,176 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T03:45:12,177 DEBUG [RS:0;237a1da887a0:39613 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52015565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=237a1da887a0/172.17.0.2:0 2024-11-23T03:45:12,183 DEBUG [RS:1;237a1da887a0:44315 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;237a1da887a0:44315 2024-11-23T03:45:12,185 INFO [RS:1;237a1da887a0:44315 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T03:45:12,186 INFO [RS:1;237a1da887a0:44315 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T03:45:12,186 DEBUG [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T03:45:12,188 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(2659): reportForDuty to master=237a1da887a0,32933,1732333510314 with port=44315, startcode=1732333511323 2024-11-23T03:45:12,193 DEBUG [RS:0;237a1da887a0:39613 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;237a1da887a0:39613 2024-11-23T03:45:12,194 INFO [RS:0;237a1da887a0:39613 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T03:45:12,194 INFO [RS:0;237a1da887a0:39613 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T03:45:12,194 DEBUG [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T03:45:12,196 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(2659): reportForDuty to master=237a1da887a0,32933,1732333510314 with port=39613, startcode=1732333511188 2024-11-23T03:45:12,201 DEBUG [RS:1;237a1da887a0:44315 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T03:45:12,201 DEBUG [RS:0;237a1da887a0:39613 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T03:45:12,212 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T03:45:12,224 INFO [master/237a1da887a0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T03:45:12,231 INFO [master/237a1da887a0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T03:45:12,236 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 237a1da887a0,32933,1732333510314 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T03:45:12,246 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/237a1da887a0:0, corePoolSize=5, maxPoolSize=5 2024-11-23T03:45:12,248 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/237a1da887a0:0, corePoolSize=5, maxPoolSize=5 2024-11-23T03:45:12,251 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/237a1da887a0:0, corePoolSize=5, maxPoolSize=5 2024-11-23T03:45:12,251 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/237a1da887a0:0, corePoolSize=5, maxPoolSize=5 2024-11-23T03:45:12,251 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/237a1da887a0:0, corePoolSize=10, maxPoolSize=10 2024-11-23T03:45:12,251 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,252 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/237a1da887a0:0, corePoolSize=2, maxPoolSize=2 2024-11-23T03:45:12,252 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,261 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732333542261 2024-11-23T03:45:12,263 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T03:45:12,264 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T03:45:12,265 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T03:45:12,265 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T03:45:12,268 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T03:45:12,268 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T03:45:12,269 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T03:45:12,269 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T03:45:12,271 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:12,271 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T03:45:12,272 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,277 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T03:45:12,279 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36197, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T03:45:12,279 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59151, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T03:45:12,280 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T03:45:12,280 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T03:45:12,285 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32933 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T03:45:12,288 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T03:45:12,289 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T03:45:12,291 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.large.0-1732333512290,5,FailOnTimeoutGroup] 2024-11-23T03:45:12,291 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32933 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T03:45:12,292 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.small.0-1732333512291,5,FailOnTimeoutGroup] 2024-11-23T03:45:12,292 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,292 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T03:45:12,294 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,294 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,301 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T03:45:12,302 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd 2024-11-23T03:45:12,315 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:12,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T03:45:12,321 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T03:45:12,321 DEBUG [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-23T03:45:12,321 DEBUG [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-23T03:45:12,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:12,322 WARN [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-23T03:45:12,322 WARN [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-23T03:45:12,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:12,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T03:45:12,328 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T03:45:12,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:12,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:12,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T03:45:12,333 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T03:45:12,333 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:12,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:12,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T03:45:12,337 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T03:45:12,338 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:12,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:12,338 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T03:45:12,339 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740 2024-11-23T03:45:12,339 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740 2024-11-23T03:45:12,340 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T03:45:12,340 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T03:45:12,340 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T03:45:12,341 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T03:45:12,347 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T03:45:12,348 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69066083, jitterRate=0.029164835810661316}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T03:45:12,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732333512315Initializing all the Stores at 1732333512316 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333512316Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333512316Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333512316Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333512316Cleaning up temporary data from old regions at 1732333512340 (+24 ms)Region opened successfully at 1732333512350 (+10 ms) 2024-11-23T03:45:12,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T03:45:12,351 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T03:45:12,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T03:45:12,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T03:45:12,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T03:45:12,353 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T03:45:12,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732333512351Disabling compacts and flushes for region at 1732333512351Disabling writes for close at 1732333512351Writing region close event to WAL at 1732333512352 (+1 ms)Closed at 1732333512352 2024-11-23T03:45:12,355 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T03:45:12,355 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T03:45:12,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T03:45:12,369 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T03:45:12,371 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T03:45:12,423 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(2659): reportForDuty to master=237a1da887a0,32933,1732333510314 with port=39613, startcode=1732333511188 2024-11-23T03:45:12,423 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(2659): reportForDuty to master=237a1da887a0,32933,1732333510314 with port=44315, startcode=1732333511323 2024-11-23T03:45:12,425 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32933 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 237a1da887a0,44315,1732333511323 2024-11-23T03:45:12,428 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32933 {}] master.ServerManager(517): Registering regionserver=237a1da887a0,44315,1732333511323 2024-11-23T03:45:12,435 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32933 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 237a1da887a0,39613,1732333511188 2024-11-23T03:45:12,435 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32933 {}] master.ServerManager(517): Registering regionserver=237a1da887a0,39613,1732333511188 2024-11-23T03:45:12,436 DEBUG [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd 2024-11-23T03:45:12,436 DEBUG [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T03:45:12,436 DEBUG [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T03:45:12,437 DEBUG [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd 2024-11-23T03:45:12,437 DEBUG [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T03:45:12,437 DEBUG [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T03:45:12,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T03:45:12,445 DEBUG [RS:0;237a1da887a0:39613 {}] zookeeper.ZKUtil(111): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/237a1da887a0,39613,1732333511188 2024-11-23T03:45:12,445 DEBUG [RS:1;237a1da887a0:44315 {}] zookeeper.ZKUtil(111): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/237a1da887a0,44315,1732333511323 2024-11-23T03:45:12,445 WARN [RS:0;237a1da887a0:39613 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T03:45:12,445 WARN [RS:1;237a1da887a0:44315 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T03:45:12,445 INFO [RS:1;237a1da887a0:44315 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T03:45:12,445 INFO [RS:0;237a1da887a0:39613 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T03:45:12,446 DEBUG [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,39613,1732333511188 2024-11-23T03:45:12,446 DEBUG [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,44315,1732333511323 2024-11-23T03:45:12,448 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [237a1da887a0,44315,1732333511323] 2024-11-23T03:45:12,448 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [237a1da887a0,39613,1732333511188] 2024-11-23T03:45:12,472 INFO [RS:1;237a1da887a0:44315 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T03:45:12,472 INFO [RS:0;237a1da887a0:39613 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T03:45:12,485 INFO [RS:1;237a1da887a0:44315 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T03:45:12,485 INFO [RS:0;237a1da887a0:39613 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T03:45:12,490 INFO [RS:1;237a1da887a0:44315 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T03:45:12,490 INFO [RS:0;237a1da887a0:39613 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T03:45:12,490 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,490 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,491 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T03:45:12,491 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T03:45:12,496 INFO [RS:1;237a1da887a0:44315 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T03:45:12,496 INFO [RS:0;237a1da887a0:39613 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T03:45:12,497 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,497 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,497 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,497 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,498 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,498 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,498 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,498 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,498 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,498 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,498 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,498 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/237a1da887a0:0, corePoolSize=2, maxPoolSize=2 2024-11-23T03:45:12,498 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,498 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,498 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,499 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/237a1da887a0:0, corePoolSize=2, maxPoolSize=2 2024-11-23T03:45:12,499 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,499 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,499 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,499 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,499 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,499 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,499 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/237a1da887a0:0, corePoolSize=3, maxPoolSize=3 2024-11-23T03:45:12,499 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,499 DEBUG [RS:1;237a1da887a0:44315 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/237a1da887a0:0, corePoolSize=3, maxPoolSize=3 2024-11-23T03:45:12,499 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,500 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,500 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:12,500 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/237a1da887a0:0, corePoolSize=3, maxPoolSize=3 2024-11-23T03:45:12,500 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,500 DEBUG [RS:0;237a1da887a0:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/237a1da887a0:0, corePoolSize=3, maxPoolSize=3 2024-11-23T03:45:12,501 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,501 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,501 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,501 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,501 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,44315,1732333511323-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T03:45:12,501 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,502 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,502 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,502 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,502 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,502 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,39613,1732333511188-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T03:45:12,523 WARN [237a1da887a0:32933 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T03:45:12,527 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T03:45:12,528 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T03:45:12,529 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,44315,1732333511323-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,529 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,39613,1732333511188-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,529 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,529 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,530 INFO [RS:1;237a1da887a0:44315 {}] regionserver.Replication(171): 237a1da887a0,44315,1732333511323 started 2024-11-23T03:45:12,530 INFO [RS:0;237a1da887a0:39613 {}] regionserver.Replication(171): 237a1da887a0,39613,1732333511188 started 2024-11-23T03:45:12,548 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,548 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(1482): Serving as 237a1da887a0,44315,1732333511323, RpcServer on 237a1da887a0/172.17.0.2:44315, sessionid=0x10142ed88000002 2024-11-23T03:45:12,549 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T03:45:12,549 DEBUG [RS:1;237a1da887a0:44315 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 237a1da887a0,44315,1732333511323 2024-11-23T03:45:12,550 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '237a1da887a0,44315,1732333511323' 2024-11-23T03:45:12,550 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T03:45:12,551 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T03:45:12,551 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T03:45:12,551 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T03:45:12,552 DEBUG [RS:1;237a1da887a0:44315 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 237a1da887a0,44315,1732333511323 2024-11-23T03:45:12,552 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '237a1da887a0,44315,1732333511323' 2024-11-23T03:45:12,552 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T03:45:12,552 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T03:45:12,553 DEBUG [RS:1;237a1da887a0:44315 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T03:45:12,553 INFO [RS:1;237a1da887a0:44315 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T03:45:12,553 INFO [RS:1;237a1da887a0:44315 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T03:45:12,555 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:12,555 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(1482): Serving as 237a1da887a0,39613,1732333511188, RpcServer on 237a1da887a0/172.17.0.2:39613, sessionid=0x10142ed88000001 2024-11-23T03:45:12,555 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T03:45:12,555 DEBUG [RS:0;237a1da887a0:39613 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 237a1da887a0,39613,1732333511188 2024-11-23T03:45:12,556 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '237a1da887a0,39613,1732333511188' 2024-11-23T03:45:12,556 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T03:45:12,556 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T03:45:12,557 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T03:45:12,557 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T03:45:12,557 DEBUG [RS:0;237a1da887a0:39613 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 237a1da887a0,39613,1732333511188 2024-11-23T03:45:12,557 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '237a1da887a0,39613,1732333511188' 2024-11-23T03:45:12,557 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T03:45:12,558 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T03:45:12,559 DEBUG [RS:0;237a1da887a0:39613 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T03:45:12,559 INFO [RS:0;237a1da887a0:39613 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T03:45:12,559 INFO [RS:0;237a1da887a0:39613 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T03:45:12,659 INFO [RS:1;237a1da887a0:44315 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T03:45:12,660 INFO [RS:0;237a1da887a0:39613 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T03:45:12,660 INFO [RS:1;237a1da887a0:44315 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=237a1da887a0%2C44315%2C1732333511323, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,44315,1732333511323, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs, maxLogs=55 2024-11-23T03:45:12,661 INFO [RS:0;237a1da887a0:39613 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=237a1da887a0%2C39613%2C1732333511188, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,39613,1732333511188, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs, maxLogs=55 2024-11-23T03:45:12,682 INFO [RS:0;237a1da887a0:39613 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,39613,1732333511188/237a1da887a0%2C39613%2C1732333511188.1732333512663 2024-11-23T03:45:12,682 INFO [RS:1;237a1da887a0:44315 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,44315,1732333511323/237a1da887a0%2C44315%2C1732333511323.1732333512663 2024-11-23T03:45:12,682 DEBUG [RS:0;237a1da887a0:39613 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T03:45:12,682 DEBUG [RS:1;237a1da887a0:44315 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T03:45:12,775 DEBUG [237a1da887a0:32933 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-23T03:45:12,787 DEBUG [237a1da887a0:32933 {}] balancer.BalancerClusterState(204): Hosts are {237a1da887a0=0} racks are {/default-rack=0} 2024-11-23T03:45:12,790 DEBUG [237a1da887a0:32933 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T03:45:12,790 DEBUG [237a1da887a0:32933 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T03:45:12,790 DEBUG [237a1da887a0:32933 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T03:45:12,791 DEBUG [237a1da887a0:32933 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T03:45:12,791 INFO [237a1da887a0:32933 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T03:45:12,791 INFO [237a1da887a0:32933 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T03:45:12,791 DEBUG [237a1da887a0:32933 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T03:45:12,798 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=237a1da887a0,44315,1732333511323 2024-11-23T03:45:12,804 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 237a1da887a0,44315,1732333511323, state=OPENING 2024-11-23T03:45:12,809 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T03:45:12,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:12,812 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:12,812 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:12,812 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:12,814 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T03:45:12,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=237a1da887a0,44315,1732333511323}] 2024-11-23T03:45:12,991 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T03:45:12,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50945, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T03:45:13,002 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T03:45:13,003 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T03:45:13,004 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T03:45:13,005 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=237a1da887a0%2C44315%2C1732333511323.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,44315,1732333511323, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs, maxLogs=55 2024-11-23T03:45:13,026 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,44315,1732333511323/237a1da887a0%2C44315%2C1732333511323.meta.1732333513006.meta 2024-11-23T03:45:13,026 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T03:45:13,027 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T03:45:13,028 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T03:45:13,031 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T03:45:13,035 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T03:45:13,039 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T03:45:13,040 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:13,040 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T03:45:13,040 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T03:45:13,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T03:45:13,043 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T03:45:13,043 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:13,043 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:13,044 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T03:45:13,045 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T03:45:13,045 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:13,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:13,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T03:45:13,046 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T03:45:13,046 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:13,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:13,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T03:45:13,047 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T03:45:13,047 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:13,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:13,048 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T03:45:13,049 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740 2024-11-23T03:45:13,049 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740 2024-11-23T03:45:13,050 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T03:45:13,050 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T03:45:13,050 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T03:45:13,051 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T03:45:13,052 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64590581, jitterRate=-0.03752534091472626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T03:45:13,052 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T03:45:13,054 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732333513041Writing region info on filesystem at 1732333513041Initializing all the Stores at 1732333513041Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333513041Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333513042 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333513042Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333513042Cleaning up temporary data from old regions at 1732333513050 (+8 ms)Running coprocessor post-open hooks at 1732333513052 (+2 ms)Region opened successfully at 1732333513053 (+1 ms) 2024-11-23T03:45:13,059 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732333512983 2024-11-23T03:45:13,069 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T03:45:13,070 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T03:45:13,071 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=237a1da887a0,44315,1732333511323 2024-11-23T03:45:13,072 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 237a1da887a0,44315,1732333511323, state=OPEN 2024-11-23T03:45:13,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T03:45:13,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T03:45:13,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T03:45:13,074 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:13,074 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:13,074 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:13,075 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=237a1da887a0,44315,1732333511323 2024-11-23T03:45:13,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T03:45:13,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=237a1da887a0,44315,1732333511323 in 259 msec 2024-11-23T03:45:13,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T03:45:13,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 717 msec 2024-11-23T03:45:13,083 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T03:45:13,083 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T03:45:13,104 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T03:45:13,105 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=237a1da887a0,44315,1732333511323, seqNum=-1] 2024-11-23T03:45:13,129 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T03:45:13,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46221, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T03:45:13,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 990 msec 2024-11-23T03:45:13,149 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732333513149, completionTime=-1 2024-11-23T03:45:13,151 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-23T03:45:13,151 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T03:45:13,177 INFO [master/237a1da887a0:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-23T03:45:13,177 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732333573177 2024-11-23T03:45:13,177 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732333633177 2024-11-23T03:45:13,177 INFO [master/237a1da887a0:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-23T03:45:13,179 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T03:45:13,186 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,32933,1732333510314-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:13,186 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,32933,1732333510314-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:13,186 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,32933,1732333510314-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:13,187 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-237a1da887a0:32933, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:13,188 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:13,188 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:13,196 DEBUG [master/237a1da887a0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T03:45:13,218 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.797sec 2024-11-23T03:45:13,220 INFO [master/237a1da887a0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T03:45:13,221 INFO [master/237a1da887a0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T03:45:13,222 INFO [master/237a1da887a0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T03:45:13,223 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T03:45:13,223 INFO [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T03:45:13,224 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,32933,1732333510314-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T03:45:13,224 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,32933,1732333510314-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T03:45:13,226 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T03:45:13,227 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T03:45:13,227 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,32933,1732333510314-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:13,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T03:45:13,270 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T03:45:13,270 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T03:45:13,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 237a1da887a0,32933,-1 for getting cluster id 2024-11-23T03:45:13,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T03:45:13,289 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b7834cd3-746d-48f8-962a-06940d656df8' 2024-11-23T03:45:13,292 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T03:45:13,292 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b7834cd3-746d-48f8-962a-06940d656df8" 2024-11-23T03:45:13,295 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aff59ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T03:45:13,295 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [237a1da887a0,32933,-1] 2024-11-23T03:45:13,298 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T03:45:13,300 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:13,302 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T03:45:13,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T03:45:13,305 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T03:45:13,313 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=237a1da887a0,44315,1732333511323, seqNum=-1] 2024-11-23T03:45:13,313 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T03:45:13,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T03:45:13,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=237a1da887a0,32933,1732333510314 2024-11-23T03:45:13,360 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T03:45:13,368 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 237a1da887a0,32933,1732333510314 2024-11-23T03:45:13,371 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5951fdf6 2024-11-23T03:45:13,372 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T03:45:13,375 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T03:45:13,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732333513355', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T03:45:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732333513355 2024-11-23T03:45:13,391 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732333513355 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T03:45:13,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732333513355" procId is: 4 2024-11-23T03:45:13,395 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:13,396 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732333513355 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T03:45:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T03:45:13,411 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732333513355-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1bc9a82a9a46ac2fc921859c2a92aea0, NAME => 'testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732333513355', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd 2024-11-23T03:45:13,419 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732333513355-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:13,419 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732333513355-pool-0 {}] regionserver.HRegion(1722): Closing 1bc9a82a9a46ac2fc921859c2a92aea0, disabling compactions & flushes 2024-11-23T03:45:13,419 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732333513355-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:13,420 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732333513355-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:13,420 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732333513355-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. after waiting 0 ms 2024-11-23T03:45:13,420 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732333513355-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:13,420 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732333513355-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:13,420 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732333513355-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1bc9a82a9a46ac2fc921859c2a92aea0: Waiting for close lock at 1732333513419Disabling compacts and flushes for region at 1732333513419Disabling writes for close at 1732333513420 (+1 ms)Writing region close event to WAL at 1732333513420Closed at 1732333513420 2024-11-23T03:45:13,421 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732333513355 execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T03:45:13,427 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732333513422"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732333513422"}]},"ts":"1732333513422"} 2024-11-23T03:45:13,431 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T03:45:13,432 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732333513355 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T03:45:13,435 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732333513355","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732333513432"}]},"ts":"1732333513432"} 2024-11-23T03:45:13,439 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732333513355, state=ENABLING in hbase:meta 2024-11-23T03:45:13,440 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {237a1da887a0=0} racks are {/default-rack=0} 2024-11-23T03:45:13,440 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T03:45:13,440 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T03:45:13,440 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T03:45:13,440 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T03:45:13,440 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T03:45:13,440 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T03:45:13,440 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T03:45:13,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732333513355, region=1bc9a82a9a46ac2fc921859c2a92aea0, ASSIGN}] 2024-11-23T03:45:13,443 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732333513355, region=1bc9a82a9a46ac2fc921859c2a92aea0, ASSIGN 2024-11-23T03:45:13,444 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732333513355, region=1bc9a82a9a46ac2fc921859c2a92aea0, ASSIGN; state=OFFLINE, location=237a1da887a0,44315,1732333511323; forceNewPlan=false, retain=false 2024-11-23T03:45:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T03:45:13,597 INFO [237a1da887a0:32933 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T03:45:13,598 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1bc9a82a9a46ac2fc921859c2a92aea0, regionState=OPENING, regionLocation=237a1da887a0,44315,1732333511323 2024-11-23T03:45:13,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732333513355, region=1bc9a82a9a46ac2fc921859c2a92aea0, ASSIGN because future has completed 2024-11-23T03:45:13,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bc9a82a9a46ac2fc921859c2a92aea0, server=237a1da887a0,44315,1732333511323}] 2024-11-23T03:45:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T03:45:13,758 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:13,759 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1bc9a82a9a46ac2fc921859c2a92aea0, NAME => 'testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T03:45:13,759 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732333513355 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:13,759 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:13,760 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:13,760 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:13,761 INFO [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:13,765 INFO [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bc9a82a9a46ac2fc921859c2a92aea0 columnFamilyName family 2024-11-23T03:45:13,765 DEBUG [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:13,765 INFO [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] regionserver.HStore(327): Store=1bc9a82a9a46ac2fc921859c2a92aea0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T03:45:13,766 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:13,766 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:13,767 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:13,767 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:13,767 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:13,768 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:13,774 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T03:45:13,774 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1bc9a82a9a46ac2fc921859c2a92aea0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65556036, jitterRate=-0.02313894033432007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T03:45:13,774 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:13,775 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1bc9a82a9a46ac2fc921859c2a92aea0: Running coprocessor pre-open hook at 1732333513760Writing region info on filesystem at 1732333513760Initializing all the Stores at 1732333513760Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333513760Cleaning up temporary data from old regions at 1732333513767 (+7 ms)Running coprocessor post-open hooks at 1732333513774 (+7 ms)Region opened successfully at 1732333513775 (+1 ms) 2024-11-23T03:45:13,776 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0., pid=6, masterSystemTime=1732333513756 2024-11-23T03:45:13,779 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:13,779 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:13,780 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1bc9a82a9a46ac2fc921859c2a92aea0, regionState=OPEN, openSeqNum=2, regionLocation=237a1da887a0,44315,1732333511323 2024-11-23T03:45:13,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bc9a82a9a46ac2fc921859c2a92aea0, server=237a1da887a0,44315,1732333511323 because future has completed 2024-11-23T03:45:13,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T03:45:13,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1bc9a82a9a46ac2fc921859c2a92aea0, server=237a1da887a0,44315,1732333511323 in 181 msec 2024-11-23T03:45:13,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T03:45:13,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732333513355, region=1bc9a82a9a46ac2fc921859c2a92aea0, ASSIGN in 345 msec 2024-11-23T03:45:13,788 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732333513355 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T03:45:13,788 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732333513355","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732333513788"}]},"ts":"1732333513788"} 2024-11-23T03:45:13,791 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732333513355, state=ENABLED in hbase:meta 2024-11-23T03:45:13,791 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732333513355 execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T03:45:13,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732333513355 in 407 msec 2024-11-23T03:45:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T03:45:14,031 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732333513355 completed 2024-11-23T03:45:14,031 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732333513355 get assigned. Timeout = 60000ms 2024-11-23T03:45:14,035 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T03:45:14,041 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732333513355 assigned to meta. Checking AM states. 2024-11-23T03:45:14,042 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T03:45:14,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732333513355 assigned. 2024-11-23T03:45:14,053 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732333513355', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0., hostname=237a1da887a0,44315,1732333511323, seqNum=2] 2024-11-23T03:45:14,065 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1bc9a82a9a46ac2fc921859c2a92aea0 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-23T03:45:14,092 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6950af7f19c94a449b8d9548327679f3 2024-11-23T03:45:14,139 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/.tmp/family/6950af7f19c94a449b8d9548327679f3 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-23T03:45:14,145 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/.tmp/family/6950af7f19c94a449b8d9548327679f3 2024-11-23T03:45:14,182 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/.tmp/family/6950af7f19c94a449b8d9548327679f3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3 2024-11-23T03:45:14,184 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3, delay=968 ms 2024-11-23T03:45:14,185 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3, entries=2, sequenceid=6, filesize=4.9 K 2024-11-23T03:45:14,191 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1bc9a82a9a46ac2fc921859c2a92aea0 in 125ms, sequenceid=6, compaction requested=false 2024-11-23T03:45:14,192 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732333513355' 2024-11-23T03:45:14,194 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1bc9a82a9a46ac2fc921859c2a92aea0: 2024-11-23T03:45:15,155 DEBUG [hfile-prefetch-1732333514184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6950af7f19c94a449b8d9548327679f3_0. 2024-11-23T03:45:15,160 DEBUG [hfile-prefetch-1732333514184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6950af7f19c94a449b8d9548327679f3_117. 2024-11-23T03:45:15,161 DEBUG [hfile-prefetch-1732333514184 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3, with total blocks 2, and data blocks 1 2024-11-23T03:45:15,161 DEBUG [hfile-prefetch-1732333514184 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-23T03:45:15,162 DEBUG [hfile-prefetch-1732333514184 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-23T03:45:15,162 DEBUG [hfile-prefetch-1732333514184 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3. Will sleep 100ms and try the verification again. 2024-11-23T03:45:15,178 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T03:45:15,178 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T03:45:15,179 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T03:45:15,179 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T03:45:15,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=1bc9a82a9a46ac2fc921859c2a92aea0, source=237a1da887a0,44315,1732333511323, destination=237a1da887a0,39613,1732333511188, warming up region on 237a1da887a0,39613,1732333511188 2024-11-23T03:45:15,209 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T03:45:15,209 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732333515179 2024-11-23T03:45:15,209 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T03:45:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T03:45:15,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=1bc9a82a9a46ac2fc921859c2a92aea0, source=237a1da887a0,44315,1732333511323, destination=237a1da887a0,39613,1732333511188, running balancer 2024-11-23T03:45:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732333513355, region=1bc9a82a9a46ac2fc921859c2a92aea0, REOPEN/MOVE 2024-11-23T03:45:15,214 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732333513355, region=1bc9a82a9a46ac2fc921859c2a92aea0, REOPEN/MOVE 2024-11-23T03:45:15,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39663, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T03:45:15,216 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1bc9a82a9a46ac2fc921859c2a92aea0, regionState=CLOSING, regionLocation=237a1da887a0,44315,1732333511323 2024-11-23T03:45:15,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1bc9a82a9a46ac2fc921859c2a92aea0, NAME => 'testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T03:45:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:15,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732333513355, region=1bc9a82a9a46ac2fc921859c2a92aea0, REOPEN/MOVE because future has completed 2024-11-23T03:45:15,221 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-23T03:45:15,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bc9a82a9a46ac2fc921859c2a92aea0, server=237a1da887a0,44315,1732333511323}] 2024-11-23T03:45:15,223 INFO [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,224 INFO [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bc9a82a9a46ac2fc921859c2a92aea0 columnFamilyName family 2024-11-23T03:45:15,224 DEBUG [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:15,231 DEBUG [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3 2024-11-23T03:45:15,232 INFO [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] regionserver.HStore(327): Store=1bc9a82a9a46ac2fc921859c2a92aea0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T03:45:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(1722): Closing 1bc9a82a9a46ac2fc921859c2a92aea0, disabling compactions & flushes 2024-11-23T03:45:15,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. after waiting 0 ms 2024-11-23T03:45:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,236 DEBUG [StoreFileCloser-1bc9a82a9a46ac2fc921859c2a92aea0-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3 2024-11-23T03:45:15,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(1676): Region close journal for 1bc9a82a9a46ac2fc921859c2a92aea0: Waiting for close lock at 1732333515232Disabling compacts and flushes for region at 1732333515232Disabling writes for close at 1732333515232Writing region close event to WAL at 1732333515237 (+5 ms)Closed at 1732333515237 2024-11-23T03:45:15,238 WARN [hfile-prefetch-1732333514184 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-23T03:45:15,239 DEBUG [hfile-prefetch-1732333514184 {}] hfile.PrefetchExecutor(121): Prefetch completed for 6950af7f19c94a449b8d9548327679f3 2024-11-23T03:45:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32933 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-23T03:45:15,380 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,380 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-23T03:45:15,382 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 1bc9a82a9a46ac2fc921859c2a92aea0, disabling compactions & flushes 2024-11-23T03:45:15,382 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,382 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,382 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. after waiting 0 ms 2024-11-23T03:45:15,383 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,385 DEBUG [StoreFileCloser-1bc9a82a9a46ac2fc921859c2a92aea0-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 6950af7f19c94a449b8d9548327679f3, starting offset: 0, end offset: 9223372036854775807 2024-11-23T03:45:15,386 DEBUG [StoreFileCloser-1bc9a82a9a46ac2fc921859c2a92aea0-family-1 {}] bucket.BucketCache(856): removed key 6950af7f19c94a449b8d9548327679f3_0 from back map with offset lock 2094080 in the evict process 2024-11-23T03:45:15,392 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-23T03:45:15,393 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,394 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 1bc9a82a9a46ac2fc921859c2a92aea0: Waiting for close lock at 1732333515382Running coprocessor pre-close hooks at 1732333515382Disabling compacts and flushes for region at 1732333515382Disabling writes for close at 1732333515382Writing region close event to WAL at 1732333515387 (+5 ms)Running coprocessor post-close hooks at 1732333515392 (+5 ms)Closed at 1732333515393 (+1 ms) 2024-11-23T03:45:15,394 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 1bc9a82a9a46ac2fc921859c2a92aea0 move to 237a1da887a0,39613,1732333511188 record at close sequenceid=6 2024-11-23T03:45:15,398 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,398 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1bc9a82a9a46ac2fc921859c2a92aea0, regionState=CLOSED 2024-11-23T03:45:15,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bc9a82a9a46ac2fc921859c2a92aea0, server=237a1da887a0,44315,1732333511323 because future has completed 2024-11-23T03:45:15,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T03:45:15,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bc9a82a9a46ac2fc921859c2a92aea0, server=237a1da887a0,44315,1732333511323 in 184 msec 2024-11-23T03:45:15,408 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732333513355, region=1bc9a82a9a46ac2fc921859c2a92aea0, REOPEN/MOVE; state=CLOSED, location=237a1da887a0,39613,1732333511188; forceNewPlan=false, retain=false 2024-11-23T03:45:15,410 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T03:45:15,410 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T03:45:15,410 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T03:45:15,410 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T03:45:15,413 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T03:45:15,413 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732333515410 2024-11-23T03:45:15,413 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T03:45:15,559 INFO [237a1da887a0:32933 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T03:45:15,559 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1bc9a82a9a46ac2fc921859c2a92aea0, regionState=OPENING, regionLocation=237a1da887a0,39613,1732333511188 2024-11-23T03:45:15,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732333513355, region=1bc9a82a9a46ac2fc921859c2a92aea0, REOPEN/MOVE because future has completed 2024-11-23T03:45:15,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bc9a82a9a46ac2fc921859c2a92aea0, server=237a1da887a0,39613,1732333511188}] 2024-11-23T03:45:15,720 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,720 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1bc9a82a9a46ac2fc921859c2a92aea0, NAME => 'testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T03:45:15,721 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732333513355 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,721 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:15,721 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,721 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,722 INFO [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,723 INFO [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bc9a82a9a46ac2fc921859c2a92aea0 columnFamilyName family 2024-11-23T03:45:15,723 DEBUG [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:15,725 DEBUG [StoreFileOpener-1bc9a82a9a46ac2fc921859c2a92aea0-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3, delay=971 ms 2024-11-23T03:45:15,726 DEBUG [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3 2024-11-23T03:45:15,726 INFO [StoreOpener-1bc9a82a9a46ac2fc921859c2a92aea0-1 {}] regionserver.HStore(327): Store=1bc9a82a9a46ac2fc921859c2a92aea0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T03:45:15,727 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,728 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,728 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,728 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,728 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,733 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,733 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1bc9a82a9a46ac2fc921859c2a92aea0; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63499650, jitterRate=-0.053781479597091675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T03:45:15,734 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,735 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1bc9a82a9a46ac2fc921859c2a92aea0: Running coprocessor pre-open hook at 1732333515721Writing region info on filesystem at 1732333515721Initializing all the Stores at 1732333515722 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333515722Cleaning up temporary data from old regions at 1732333515728 (+6 ms)Running coprocessor post-open hooks at 1732333515734 (+6 ms)Region opened successfully at 1732333515735 (+1 ms) 2024-11-23T03:45:15,736 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0., pid=9, masterSystemTime=1732333515717 2024-11-23T03:45:15,739 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,739 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,739 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1bc9a82a9a46ac2fc921859c2a92aea0, regionState=OPEN, openSeqNum=10, regionLocation=237a1da887a0,39613,1732333511188 2024-11-23T03:45:15,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bc9a82a9a46ac2fc921859c2a92aea0, server=237a1da887a0,39613,1732333511188 because future has completed 2024-11-23T03:45:15,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-23T03:45:15,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 1bc9a82a9a46ac2fc921859c2a92aea0, server=237a1da887a0,39613,1732333511188 in 179 msec 2024-11-23T03:45:15,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732333513355, region=1bc9a82a9a46ac2fc921859c2a92aea0, REOPEN/MOVE in 535 msec 2024-11-23T03:45:15,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T03:45:15,767 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T03:45:15,767 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T03:45:15,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:15,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:15,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T03:45:15,773 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T03:45:15,773 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-23T03:45:15,773 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=237a1da887a0,32933,1732333510314 2024-11-23T03:45:15,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:15,776 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T03:45:15,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:15,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:15,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:15,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:15,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:15,777 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T03:45:15,777 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T03:45:15,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:15,777 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:15,777 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:15,777 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '237a1da887a0,39613,1732333511188' ***** 2024-11-23T03:45:15,778 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T03:45:15,778 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '237a1da887a0,44315,1732333511323' ***** 2024-11-23T03:45:15,778 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T03:45:15,778 INFO [RS:0;237a1da887a0:39613 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T03:45:15,778 INFO [RS:1;237a1da887a0:44315 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T03:45:15,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:15,778 INFO [RS:1;237a1da887a0:44315 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/bucket.persistence 2024-11-23T03:45:15,778 INFO [RS:0;237a1da887a0:39613 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/bucket.persistence 2024-11-23T03:45:15,778 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T03:45:15,778 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T03:45:15,778 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T03:45:15,779 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T03:45:15,778 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T03:45:15,778 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T03:45:15,779 INFO [RS:0;237a1da887a0:39613 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T03:45:15,779 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T03:45:15,780 INFO [RS:1;237a1da887a0:44315 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T03:45:15,780 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T03:45:15,789 DEBUG [RS:0;237a1da887a0:39613 {}] bucket.BucketCache(1487): Thread RS:0;237a1da887a0:39613 started persisting bucket cache to file 2024-11-23T03:45:15,790 DEBUG [RS:1;237a1da887a0:44315 {}] bucket.BucketCache(1487): Thread RS:1;237a1da887a0:44315 started persisting bucket cache to file 2024-11-23T03:45:15,790 DEBUG [RS:0;237a1da887a0:39613 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T03:45:15,790 DEBUG [RS:0;237a1da887a0:39613 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T03:45:15,790 DEBUG [RS:1;237a1da887a0:44315 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T03:45:15,790 DEBUG [RS:1;237a1da887a0:44315 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T03:45:15,794 DEBUG [RS:0;237a1da887a0:39613 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T03:45:15,794 DEBUG [RS:0;237a1da887a0:39613 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732333515789 2024-11-23T03:45:15,794 DEBUG [RS:0;237a1da887a0:39613 {}] bucket.BucketCache(1509): Thread RS:0;237a1da887a0:39613 finished persisting bucket cache to file, renaming 2024-11-23T03:45:15,794 DEBUG [RS:1;237a1da887a0:44315 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T03:45:15,794 DEBUG [RS:1;237a1da887a0:44315 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732333515790 2024-11-23T03:45:15,794 DEBUG [RS:1;237a1da887a0:44315 {}] bucket.BucketCache(1509): Thread RS:1;237a1da887a0:44315 finished persisting bucket cache to file, renaming 2024-11-23T03:45:15,794 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T03:45:15,794 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T03:45:15,794 INFO [RS:0;237a1da887a0:39613 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T03:45:15,794 INFO [RS:0;237a1da887a0:39613 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T03:45:15,795 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T03:45:15,795 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(3091): Received CLOSE for 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,795 INFO [RS:1;237a1da887a0:44315 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T03:45:15,795 INFO [RS:1;237a1da887a0:44315 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T03:45:15,795 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(959): stopping server 237a1da887a0,44315,1732333511323 2024-11-23T03:45:15,795 INFO [RS:1;237a1da887a0:44315 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T03:45:15,795 INFO [RS:1;237a1da887a0:44315 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;237a1da887a0:44315. 2024-11-23T03:45:15,795 DEBUG [RS:1;237a1da887a0:44315 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T03:45:15,795 DEBUG [RS:1;237a1da887a0:44315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:15,795 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(959): stopping server 237a1da887a0,39613,1732333511188 2024-11-23T03:45:15,795 INFO [RS:0;237a1da887a0:39613 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T03:45:15,795 INFO [RS:0;237a1da887a0:39613 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;237a1da887a0:39613. 2024-11-23T03:45:15,795 INFO [RS:1;237a1da887a0:44315 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T03:45:15,795 INFO [RS:1;237a1da887a0:44315 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T03:45:15,795 DEBUG [RS:0;237a1da887a0:39613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T03:45:15,795 INFO [RS:1;237a1da887a0:44315 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T03:45:15,796 DEBUG [RS:0;237a1da887a0:39613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:15,796 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T03:45:15,796 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T03:45:15,796 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T03:45:15,796 DEBUG [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(1325): Online Regions={1bc9a82a9a46ac2fc921859c2a92aea0=testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0.} 2024-11-23T03:45:15,796 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T03:45:15,796 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T03:45:15,796 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1bc9a82a9a46ac2fc921859c2a92aea0, disabling compactions & flushes 2024-11-23T03:45:15,796 DEBUG [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-23T03:45:15,796 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,796 DEBUG [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(1351): Waiting on 1bc9a82a9a46ac2fc921859c2a92aea0 2024-11-23T03:45:15,796 DEBUG [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T03:45:15,796 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,796 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T03:45:15,796 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. after waiting 0 ms 2024-11-23T03:45:15,797 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,797 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T03:45:15,797 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T03:45:15,797 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T03:45:15,797 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T03:45:15,797 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T03:45:15,797 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-23T03:45:15,797 DEBUG [StoreFileCloser-1bc9a82a9a46ac2fc921859c2a92aea0-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/family/6950af7f19c94a449b8d9548327679f3 2024-11-23T03:45:15,804 INFO [regionserver/237a1da887a0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T03:45:15,805 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnRegionMove1732333513355/1bc9a82a9a46ac2fc921859c2a92aea0/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-23T03:45:15,805 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,805 INFO [regionserver/237a1da887a0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T03:45:15,806 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1bc9a82a9a46ac2fc921859c2a92aea0: Waiting for close lock at 1732333515795Running coprocessor pre-close hooks at 1732333515796 (+1 ms)Disabling compacts and flushes for region at 1732333515796Disabling writes for close at 1732333515796Writing region close event to WAL at 1732333515798 (+2 ms)Running coprocessor post-close hooks at 1732333515805 (+7 ms)Closed at 1732333515805 2024-11-23T03:45:15,806 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0. 2024-11-23T03:45:15,809 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0c924940c4064ad7b44e75bb41914c7a 2024-11-23T03:45:15,827 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/info/0c924940c4064ad7b44e75bb41914c7a is 205, key is testBlockEvictionOnRegionMove1732333513355,,1732333513377.1bc9a82a9a46ac2fc921859c2a92aea0./info:regioninfo/1732333515739/Put/seqid=0 2024-11-23T03:45:15,828 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/info/0c924940c4064ad7b44e75bb41914c7a 2024-11-23T03:45:15,843 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b613c1ae714b49a6be307f953f3d8043 2024-11-23T03:45:15,859 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/ns/b613c1ae714b49a6be307f953f3d8043 is 43, key is default/ns:d/1732333513136/Put/seqid=0 2024-11-23T03:45:15,860 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/ns/b613c1ae714b49a6be307f953f3d8043 2024-11-23T03:45:15,871 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8d1415f27b724540990abf1465b6e0e0 2024-11-23T03:45:15,884 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/table/8d1415f27b724540990abf1465b6e0e0 is 78, key is testBlockEvictionOnRegionMove1732333513355/table:state/1732333513788/Put/seqid=0 2024-11-23T03:45:15,884 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/table/8d1415f27b724540990abf1465b6e0e0 2024-11-23T03:45:15,887 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/info/0c924940c4064ad7b44e75bb41914c7a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/0c924940c4064ad7b44e75bb41914c7a 2024-11-23T03:45:15,888 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/0c924940c4064ad7b44e75bb41914c7a, delay=964 ms 2024-11-23T03:45:15,889 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/0c924940c4064ad7b44e75bb41914c7a, entries=15, sequenceid=15, filesize=7.6 K 2024-11-23T03:45:15,890 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/ns/b613c1ae714b49a6be307f953f3d8043 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/ns/b613c1ae714b49a6be307f953f3d8043 2024-11-23T03:45:15,891 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/ns/b613c1ae714b49a6be307f953f3d8043, delay=978 ms 2024-11-23T03:45:15,892 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/ns/b613c1ae714b49a6be307f953f3d8043, entries=2, sequenceid=15, filesize=5.0 K 2024-11-23T03:45:15,892 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/table/8d1415f27b724540990abf1465b6e0e0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/table/8d1415f27b724540990abf1465b6e0e0 2024-11-23T03:45:15,894 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/table/8d1415f27b724540990abf1465b6e0e0, delay=962 ms 2024-11-23T03:45:15,894 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/table/8d1415f27b724540990abf1465b6e0e0, entries=2, sequenceid=15, filesize=5.3 K 2024-11-23T03:45:15,895 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 98ms, sequenceid=15, compaction requested=false 2024-11-23T03:45:15,895 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T03:45:15,897 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/0c924940c4064ad7b44e75bb41914c7a 2024-11-23T03:45:15,897 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/ns/b613c1ae714b49a6be307f953f3d8043 2024-11-23T03:45:15,898 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/table/8d1415f27b724540990abf1465b6e0e0 2024-11-23T03:45:15,902 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-23T03:45:15,903 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T03:45:15,903 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T03:45:15,903 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732333515796Running coprocessor pre-close hooks at 1732333515796Disabling compacts and flushes for region at 1732333515797 (+1 ms)Disabling writes for close at 1732333515797Obtaining lock to block concurrent updates at 1732333515797Preparing flush snapshotting stores in 1588230740 at 1732333515797Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732333515798 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732333515798Flushing 1588230740/info: creating writer at 1732333515798Flushing 1588230740/info: appending metadata at 1732333515826 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732333515826Flushing 1588230740/ns: creating writer at 1732333515831 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732333515858 (+27 ms)Flushing 1588230740/ns: closing flushed file at 1732333515858Flushing 1588230740/table: creating writer at 1732333515864 (+6 ms)Flushing 1588230740/table: appending metadata at 1732333515883 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732333515883Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36003ebe: reopening flushed file at 1732333515887 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b68c844: reopening flushed file at 1732333515890 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67a4c51c: reopening flushed file at 1732333515892 (+2 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 98ms, sequenceid=15, compaction requested=false at 1732333515895 (+3 ms)Writing region close event to WAL at 1732333515898 (+3 ms)Running coprocessor post-close hooks at 1732333515903 (+5 ms)Closed at 1732333515903 2024-11-23T03:45:15,904 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T03:45:15,997 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(976): stopping server 237a1da887a0,44315,1732333511323; all regions closed. 2024-11-23T03:45:15,997 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(976): stopping server 237a1da887a0,39613,1732333511188; all regions closed. 2024-11-23T03:45:16,003 DEBUG [RS:0;237a1da887a0:39613 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs 2024-11-23T03:45:16,003 DEBUG [RS:1;237a1da887a0:44315 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs 2024-11-23T03:45:16,003 INFO [RS:0;237a1da887a0:39613 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 237a1da887a0%2C39613%2C1732333511188:(num 1732333512663) 2024-11-23T03:45:16,003 DEBUG [RS:0;237a1da887a0:39613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:16,003 INFO [RS:0;237a1da887a0:39613 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T03:45:16,003 INFO [RS:1;237a1da887a0:44315 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 237a1da887a0%2C44315%2C1732333511323.meta:.meta(num 1732333513006) 2024-11-23T03:45:16,003 INFO [RS:0;237a1da887a0:39613 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T03:45:16,003 INFO [RS:0;237a1da887a0:39613 {}] hbase.ChoreService(370): Chore service for: regionserver/237a1da887a0:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T03:45:16,004 INFO [RS:0;237a1da887a0:39613 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T03:45:16,004 INFO [RS:0;237a1da887a0:39613 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T03:45:16,004 INFO [RS:0;237a1da887a0:39613 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T03:45:16,004 INFO [RS:0;237a1da887a0:39613 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T03:45:16,004 INFO [regionserver/237a1da887a0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T03:45:16,004 INFO [RS:0;237a1da887a0:39613 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39613 2024-11-23T03:45:16,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T03:45:16,009 INFO [RS:0;237a1da887a0:39613 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T03:45:16,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/237a1da887a0,39613,1732333511188 2024-11-23T03:45:16,011 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [237a1da887a0,39613,1732333511188] 2024-11-23T03:45:16,012 DEBUG [RS:1;237a1da887a0:44315 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs 2024-11-23T03:45:16,012 INFO [RS:1;237a1da887a0:44315 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 237a1da887a0%2C44315%2C1732333511323:(num 1732333512663) 2024-11-23T03:45:16,012 DEBUG [RS:1;237a1da887a0:44315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:16,012 INFO [RS:1;237a1da887a0:44315 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T03:45:16,012 INFO [RS:1;237a1da887a0:44315 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T03:45:16,013 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/237a1da887a0,39613,1732333511188 already deleted, retry=false 2024-11-23T03:45:16,013 INFO [RS:1;237a1da887a0:44315 {}] hbase.ChoreService(370): Chore service for: regionserver/237a1da887a0:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T03:45:16,013 INFO [RS:1;237a1da887a0:44315 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T03:45:16,013 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 237a1da887a0,39613,1732333511188 expired; onlineServers=1 2024-11-23T03:45:16,013 INFO [regionserver/237a1da887a0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T03:45:16,013 INFO [RS:1;237a1da887a0:44315 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44315 2024-11-23T03:45:16,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T03:45:16,015 INFO [RS:1;237a1da887a0:44315 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T03:45:16,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/237a1da887a0,44315,1732333511323 2024-11-23T03:45:16,016 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [237a1da887a0,44315,1732333511323] 2024-11-23T03:45:16,019 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/237a1da887a0,44315,1732333511323 already deleted, retry=false 2024-11-23T03:45:16,019 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 237a1da887a0,44315,1732333511323 expired; onlineServers=0 2024-11-23T03:45:16,019 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '237a1da887a0,32933,1732333510314' ***** 2024-11-23T03:45:16,019 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T03:45:16,020 INFO [M:0;237a1da887a0:32933 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T03:45:16,020 INFO [M:0;237a1da887a0:32933 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T03:45:16,020 DEBUG [M:0;237a1da887a0:32933 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T03:45:16,020 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T03:45:16,020 DEBUG [M:0;237a1da887a0:32933 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T03:45:16,020 DEBUG [master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.small.0-1732333512291 {}] cleaner.HFileCleaner(306): Exit Thread[master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.small.0-1732333512291,5,FailOnTimeoutGroup] 2024-11-23T03:45:16,020 DEBUG [master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.large.0-1732333512290 {}] cleaner.HFileCleaner(306): Exit Thread[master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.large.0-1732333512290,5,FailOnTimeoutGroup] 2024-11-23T03:45:16,020 INFO [M:0;237a1da887a0:32933 {}] hbase.ChoreService(370): Chore service for: master/237a1da887a0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T03:45:16,021 INFO [M:0;237a1da887a0:32933 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T03:45:16,021 DEBUG [M:0;237a1da887a0:32933 {}] master.HMaster(1795): Stopping service threads 2024-11-23T03:45:16,021 INFO [M:0;237a1da887a0:32933 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T03:45:16,021 INFO [M:0;237a1da887a0:32933 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T03:45:16,022 INFO [M:0;237a1da887a0:32933 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T03:45:16,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T03:45:16,022 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T03:45:16,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,023 DEBUG [M:0;237a1da887a0:32933 {}] zookeeper.ZKUtil(347): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T03:45:16,023 WARN [M:0;237a1da887a0:32933 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T03:45:16,023 INFO [M:0;237a1da887a0:32933 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/.lastflushedseqids 2024-11-23T03:45:16,033 INFO [M:0;237a1da887a0:32933 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T03:45:16,033 INFO [M:0;237a1da887a0:32933 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T03:45:16,034 DEBUG [M:0;237a1da887a0:32933 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T03:45:16,034 INFO [M:0;237a1da887a0:32933 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:16,034 DEBUG [M:0;237a1da887a0:32933 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:16,034 DEBUG [M:0;237a1da887a0:32933 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T03:45:16,034 DEBUG [M:0;237a1da887a0:32933 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:16,034 INFO [M:0;237a1da887a0:32933 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-23T03:45:16,052 DEBUG [M:0;237a1da887a0:32933 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 09be9435a4ee4b4c84055a790bf857c6 2024-11-23T03:45:16,066 DEBUG [M:0;237a1da887a0:32933 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09be9435a4ee4b4c84055a790bf857c6 is 82, key is hbase:meta,,1/info:regioninfo/1732333513070/Put/seqid=0 2024-11-23T03:45:16,067 INFO [M:0;237a1da887a0:32933 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09be9435a4ee4b4c84055a790bf857c6 2024-11-23T03:45:16,077 DEBUG [M:0;237a1da887a0:32933 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5241821d65bb4d5f8a1a8a7f403c52c5 2024-11-23T03:45:16,092 DEBUG [M:0;237a1da887a0:32933 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5241821d65bb4d5f8a1a8a7f403c52c5 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732333513793/Put/seqid=0 2024-11-23T03:45:16,093 INFO [M:0;237a1da887a0:32933 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5241821d65bb4d5f8a1a8a7f403c52c5 2024-11-23T03:45:16,104 DEBUG [M:0;237a1da887a0:32933 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 49840fb652f945b09c532f7d2615d5f2 2024-11-23T03:45:16,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:16,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x10142ed88000001, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:16,114 INFO [RS:0;237a1da887a0:39613 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T03:45:16,114 INFO [RS:0;237a1da887a0:39613 {}] regionserver.HRegionServer(1031): Exiting; stopping=237a1da887a0,39613,1732333511188; zookeeper connection closed. 2024-11-23T03:45:16,114 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e6d45fe {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e6d45fe 2024-11-23T03:45:16,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:16,118 INFO [RS:1;237a1da887a0:44315 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T03:45:16,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44315-0x10142ed88000002, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:16,118 INFO [RS:1;237a1da887a0:44315 {}] regionserver.HRegionServer(1031): Exiting; stopping=237a1da887a0,44315,1732333511323; zookeeper connection closed. 2024-11-23T03:45:16,119 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ecfe8c6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ecfe8c6 2024-11-23T03:45:16,119 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-23T03:45:16,124 DEBUG [M:0;237a1da887a0:32933 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/49840fb652f945b09c532f7d2615d5f2 is 69, key is 237a1da887a0,39613,1732333511188/rs:state/1732333512436/Put/seqid=0 2024-11-23T03:45:16,125 INFO [M:0;237a1da887a0:32933 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/49840fb652f945b09c532f7d2615d5f2 2024-11-23T03:45:16,128 DEBUG [M:0;237a1da887a0:32933 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09be9435a4ee4b4c84055a790bf857c6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09be9435a4ee4b4c84055a790bf857c6 2024-11-23T03:45:16,131 INFO [M:0;237a1da887a0:32933 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09be9435a4ee4b4c84055a790bf857c6, entries=8, sequenceid=84, filesize=5.5 K 2024-11-23T03:45:16,131 DEBUG [M:0;237a1da887a0:32933 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5241821d65bb4d5f8a1a8a7f403c52c5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5241821d65bb4d5f8a1a8a7f403c52c5 2024-11-23T03:45:16,134 INFO [M:0;237a1da887a0:32933 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5241821d65bb4d5f8a1a8a7f403c52c5, entries=9, sequenceid=84, filesize=6.5 K 2024-11-23T03:45:16,134 DEBUG [M:0;237a1da887a0:32933 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/49840fb652f945b09c532f7d2615d5f2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/49840fb652f945b09c532f7d2615d5f2 2024-11-23T03:45:16,137 INFO [M:0;237a1da887a0:32933 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/49840fb652f945b09c532f7d2615d5f2, entries=2, sequenceid=84, filesize=5.1 K 2024-11-23T03:45:16,138 INFO [M:0;237a1da887a0:32933 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=84, compaction requested=false 2024-11-23T03:45:16,152 INFO [M:0;237a1da887a0:32933 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:16,153 DEBUG [M:0;237a1da887a0:32933 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732333516033Disabling compacts and flushes for region at 1732333516033Disabling writes for close at 1732333516034 (+1 ms)Obtaining lock to block concurrent updates at 1732333516034Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732333516034Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732333516034Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732333516035 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732333516035Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732333516066 (+31 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732333516066Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732333516069 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732333516091 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732333516091Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732333516096 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732333516123 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732333516123Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35f5df7b: reopening flushed file at 1732333516128 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@655047b1: reopening flushed file at 1732333516131 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ffc83c9: reopening flushed file at 1732333516134 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=84, compaction requested=false at 1732333516138 (+4 ms)Writing region close event to WAL at 1732333516152 (+14 ms)Closed at 1732333516152 2024-11-23T03:45:16,155 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T03:45:16,155 INFO [M:0;237a1da887a0:32933 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T03:45:16,155 INFO [M:0;237a1da887a0:32933 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32933 2024-11-23T03:45:16,156 INFO [M:0;237a1da887a0:32933 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T03:45:16,258 INFO [M:0;237a1da887a0:32933 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T03:45:16,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:16,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32933-0x10142ed88000000, quorum=127.0.0.1:60839, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:16,266 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T03:45:16,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T03:45:16,290 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:16,302 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/237a1da887a0:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732333515888 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732333514184 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732333515725 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/237a1da887a0:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/237a1da887a0:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732333515891 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/237a1da887a0:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 368), ProcessCount=11 (was 11), AvailableMemoryMB=10003 (was 7523) - AvailableMemoryMB LEAK? - 2024-11-23T03:45:16,308 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=334, ProcessCount=11, AvailableMemoryMB=10002 2024-11-23T03:45:16,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/hadoop-log-dir so I do NOT create it in target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b 2024-11-23T03:45:16,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/hadoop-tmp-dir so I do NOT create it in target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b 2024-11-23T03:45:16,309 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:16,312 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/cluster_59cc8b64-5989-e046-254c-2631cab10535, deleteOnExit=true 2024-11-23T03:45:16,317 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/cluster_59cc8b64-5989-e046-254c-2631cab10535/zookeeper_0, clientPort=49805, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/cluster_59cc8b64-5989-e046-254c-2631cab10535/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/cluster_59cc8b64-5989-e046-254c-2631cab10535/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T03:45:16,320 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49805 2024-11-23T03:45:16,333 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd with version=8 2024-11-23T03:45:16,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/hbase-staging 2024-11-23T03:45:16,336 INFO [Time-limited test {}] client.ConnectionUtils(128): master/237a1da887a0:0 server-side Connection retries=45 2024-11-23T03:45:16,336 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:16,337 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:16,337 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T03:45:16,337 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:16,337 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T03:45:16,337 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T03:45:16,337 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T03:45:16,338 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39357 2024-11-23T03:45:16,339 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39357 connecting to ZooKeeper ensemble=127.0.0.1:49805 2024-11-23T03:45:16,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:393570x0, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T03:45:16,347 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39357-0x10142eda0ac0000 connected 2024-11-23T03:45:16,373 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:16,373 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:16,374 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:16,374 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd, hbase.cluster.distributed=false 2024-11-23T03:45:16,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T03:45:16,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39357 2024-11-23T03:45:16,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39357 2024-11-23T03:45:16,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39357 2024-11-23T03:45:16,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39357 2024-11-23T03:45:16,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39357 2024-11-23T03:45:16,397 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/237a1da887a0:0 server-side Connection retries=45 2024-11-23T03:45:16,397 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:16,397 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:16,397 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T03:45:16,397 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:16,397 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T03:45:16,397 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T03:45:16,397 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T03:45:16,398 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44421 2024-11-23T03:45:16,399 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44421 connecting to ZooKeeper ensemble=127.0.0.1:49805 2024-11-23T03:45:16,399 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:16,400 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:16,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444210x0, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T03:45:16,403 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:444210x0, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:16,404 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T03:45:16,404 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44421-0x10142eda0ac0001 connected 2024-11-23T03:45:16,405 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.cache 2024-11-23T03:45:16,405 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T03:45:16,405 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T03:45:16,405 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T03:45:16,405 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T03:45:16,405 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T03:45:16,406 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T03:45:16,406 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.persistence is complete. 2024-11-23T03:45:16,406 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T03:45:16,407 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T03:45:16,408 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T03:45:16,409 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T03:45:16,411 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44421 2024-11-23T03:45:16,411 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44421 2024-11-23T03:45:16,411 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44421 2024-11-23T03:45:16,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44421 2024-11-23T03:45:16,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44421 2024-11-23T03:45:16,429 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/237a1da887a0:0 server-side Connection retries=45 2024-11-23T03:45:16,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:16,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:16,429 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T03:45:16,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:16,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T03:45:16,429 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T03:45:16,429 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T03:45:16,430 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35247 2024-11-23T03:45:16,432 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35247 connecting to ZooKeeper ensemble=127.0.0.1:49805 2024-11-23T03:45:16,432 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:16,432 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:16,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352470x0, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T03:45:16,437 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35247-0x10142eda0ac0002 connected 2024-11-23T03:45:16,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:16,437 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T03:45:16,438 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.cache 2024-11-23T03:45:16,438 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T03:45:16,438 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T03:45:16,439 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T03:45:16,439 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T03:45:16,439 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T03:45:16,439 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T03:45:16,439 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T03:45:16,439 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T03:45:16,439 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.persistence is complete. 2024-11-23T03:45:16,440 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T03:45:16,442 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T03:45:16,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35247 2024-11-23T03:45:16,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35247 2024-11-23T03:45:16,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35247 2024-11-23T03:45:16,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35247 2024-11-23T03:45:16,447 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35247 2024-11-23T03:45:16,461 DEBUG [M:0;237a1da887a0:39357 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;237a1da887a0:39357 2024-11-23T03:45:16,461 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/237a1da887a0,39357,1732333516336 2024-11-23T03:45:16,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:16,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:16,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:16,466 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/237a1da887a0,39357,1732333516336 2024-11-23T03:45:16,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T03:45:16,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T03:45:16,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,469 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T03:45:16,471 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/237a1da887a0,39357,1732333516336 from backup master directory 2024-11-23T03:45:16,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:16,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/237a1da887a0,39357,1732333516336 2024-11-23T03:45:16,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:16,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T03:45:16,473 WARN [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T03:45:16,473 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=237a1da887a0,39357,1732333516336 2024-11-23T03:45:16,475 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/hbase.id] with ID: 2c75c6a7-08bf-43b5-a5b3-05b5621cd0c5 2024-11-23T03:45:16,475 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/.tmp/hbase.id 2024-11-23T03:45:16,483 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/hbase.id] 2024-11-23T03:45:16,536 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:16,536 INFO [master/237a1da887a0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T03:45:16,539 INFO [master/237a1da887a0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-23T03:45:16,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,549 INFO [master/237a1da887a0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T03:45:16,551 INFO [master/237a1da887a0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T03:45:16,551 INFO [master/237a1da887a0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T03:45:16,580 INFO [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store 2024-11-23T03:45:16,587 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:16,587 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T03:45:16,587 INFO [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:16,587 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:16,588 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T03:45:16,588 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:16,588 INFO [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:16,588 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732333516587Disabling compacts and flushes for region at 1732333516587Disabling writes for close at 1732333516588 (+1 ms)Writing region close event to WAL at 1732333516588Closed at 1732333516588 2024-11-23T03:45:16,588 WARN [master/237a1da887a0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/.initializing 2024-11-23T03:45:16,588 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/WALs/237a1da887a0,39357,1732333516336 2024-11-23T03:45:16,594 INFO [master/237a1da887a0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T03:45:16,597 INFO [master/237a1da887a0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=237a1da887a0%2C39357%2C1732333516336, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/WALs/237a1da887a0,39357,1732333516336, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/oldWALs, maxLogs=10 2024-11-23T03:45:16,616 INFO [master/237a1da887a0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/WALs/237a1da887a0,39357,1732333516336/237a1da887a0%2C39357%2C1732333516336.1732333516598 2024-11-23T03:45:16,617 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T03:45:16,617 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T03:45:16,617 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:16,618 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,618 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T03:45:16,625 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:16,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:16,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T03:45:16,630 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:16,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T03:45:16,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T03:45:16,637 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:16,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T03:45:16,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T03:45:16,642 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:16,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T03:45:16,643 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,643 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,643 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,644 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,644 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,644 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T03:45:16,645 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T03:45:16,654 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T03:45:16,655 INFO [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71789908, jitterRate=0.06975299119949341}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T03:45:16,656 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732333516618Initializing all the Stores at 1732333516618Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333516618Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333516619 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333516619Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333516619Cleaning up temporary data from old regions at 1732333516644 (+25 ms)Region opened successfully at 1732333516655 (+11 ms) 2024-11-23T03:45:16,656 INFO [master/237a1da887a0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T03:45:16,665 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ccc01d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=237a1da887a0/172.17.0.2:0 2024-11-23T03:45:16,667 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T03:45:16,668 INFO [master/237a1da887a0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T03:45:16,668 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T03:45:16,668 INFO [master/237a1da887a0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T03:45:16,668 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T03:45:16,669 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-23T03:45:16,669 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T03:45:16,671 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T03:45:16,672 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T03:45:16,675 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T03:45:16,676 INFO [master/237a1da887a0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T03:45:16,677 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T03:45:16,679 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T03:45:16,679 INFO [master/237a1da887a0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T03:45:16,680 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T03:45:16,682 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T03:45:16,682 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T03:45:16,684 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T03:45:16,686 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T03:45:16,690 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T03:45:16,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:16,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:16,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:16,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,693 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=237a1da887a0,39357,1732333516336, sessionid=0x10142eda0ac0000, setting cluster-up flag (Was=false) 2024-11-23T03:45:16,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,706 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T03:45:16,707 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=237a1da887a0,39357,1732333516336 2024-11-23T03:45:16,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:16,725 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T03:45:16,728 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=237a1da887a0,39357,1732333516336 2024-11-23T03:45:16,729 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T03:45:16,734 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T03:45:16,734 INFO [master/237a1da887a0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T03:45:16,734 INFO [master/237a1da887a0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T03:45:16,735 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 237a1da887a0,39357,1732333516336 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T03:45:16,737 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/237a1da887a0:0, corePoolSize=5, maxPoolSize=5 2024-11-23T03:45:16,737 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/237a1da887a0:0, corePoolSize=5, maxPoolSize=5 2024-11-23T03:45:16,737 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/237a1da887a0:0, corePoolSize=5, maxPoolSize=5 2024-11-23T03:45:16,737 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/237a1da887a0:0, corePoolSize=5, maxPoolSize=5 2024-11-23T03:45:16,737 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/237a1da887a0:0, corePoolSize=10, maxPoolSize=10 2024-11-23T03:45:16,737 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,737 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/237a1da887a0:0, corePoolSize=2, maxPoolSize=2 2024-11-23T03:45:16,737 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,742 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T03:45:16,742 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T03:45:16,743 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:16,743 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T03:45:16,763 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(746): ClusterId : 2c75c6a7-08bf-43b5-a5b3-05b5621cd0c5 2024-11-23T03:45:16,763 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T03:45:16,767 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(746): ClusterId : 2c75c6a7-08bf-43b5-a5b3-05b5621cd0c5 2024-11-23T03:45:16,767 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T03:45:16,767 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T03:45:16,767 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T03:45:16,772 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T03:45:16,772 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T03:45:16,773 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732333546773 2024-11-23T03:45:16,773 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T03:45:16,774 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T03:45:16,774 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T03:45:16,774 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T03:45:16,774 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T03:45:16,774 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T03:45:16,775 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T03:45:16,776 DEBUG [RS:1;237a1da887a0:35247 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd5a243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=237a1da887a0/172.17.0.2:0 2024-11-23T03:45:16,776 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,776 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T03:45:16,777 DEBUG [RS:0;237a1da887a0:44421 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dae0518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=237a1da887a0/172.17.0.2:0 2024-11-23T03:45:16,784 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T03:45:16,784 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T03:45:16,784 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T03:45:16,785 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T03:45:16,785 INFO [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T03:45:16,785 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.large.0-1732333516785,5,FailOnTimeoutGroup] 2024-11-23T03:45:16,785 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.small.0-1732333516785,5,FailOnTimeoutGroup] 2024-11-23T03:45:16,785 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,787 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T03:45:16,787 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,787 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,799 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T03:45:16,800 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd 2024-11-23T03:45:16,802 DEBUG [RS:0;237a1da887a0:44421 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;237a1da887a0:44421 2024-11-23T03:45:16,802 DEBUG [RS:1;237a1da887a0:35247 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;237a1da887a0:35247 2024-11-23T03:45:16,802 INFO [RS:0;237a1da887a0:44421 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T03:45:16,802 INFO [RS:0;237a1da887a0:44421 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T03:45:16,802 INFO [RS:1;237a1da887a0:35247 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T03:45:16,802 DEBUG [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T03:45:16,802 INFO [RS:1;237a1da887a0:35247 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T03:45:16,802 DEBUG [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T03:45:16,803 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(2659): reportForDuty to master=237a1da887a0,39357,1732333516336 with port=35247, startcode=1732333516428 2024-11-23T03:45:16,803 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(2659): reportForDuty to master=237a1da887a0,39357,1732333516336 with port=44421, startcode=1732333516396 2024-11-23T03:45:16,803 DEBUG [RS:1;237a1da887a0:35247 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T03:45:16,803 DEBUG [RS:0;237a1da887a0:44421 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T03:45:16,810 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38683, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T03:45:16,810 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 237a1da887a0,35247,1732333516428 2024-11-23T03:45:16,811 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39357 {}] master.ServerManager(517): Registering regionserver=237a1da887a0,35247,1732333516428 2024-11-23T03:45:16,813 DEBUG [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd 2024-11-23T03:45:16,813 DEBUG [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T03:45:16,813 DEBUG [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T03:45:16,817 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41501, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T03:45:16,817 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 237a1da887a0,44421,1732333516396 2024-11-23T03:45:16,818 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39357 {}] master.ServerManager(517): Registering regionserver=237a1da887a0,44421,1732333516396 2024-11-23T03:45:16,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T03:45:16,820 DEBUG [RS:1;237a1da887a0:35247 {}] zookeeper.ZKUtil(111): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/237a1da887a0,35247,1732333516428 2024-11-23T03:45:16,820 WARN [RS:1;237a1da887a0:35247 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T03:45:16,820 INFO [RS:1;237a1da887a0:35247 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T03:45:16,820 DEBUG [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,35247,1732333516428 2024-11-23T03:45:16,821 DEBUG [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd 2024-11-23T03:45:16,821 DEBUG [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T03:45:16,821 DEBUG [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T03:45:16,829 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:16,830 DEBUG [RS:0;237a1da887a0:44421 {}] zookeeper.ZKUtil(111): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/237a1da887a0,44421,1732333516396 2024-11-23T03:45:16,830 WARN [RS:0;237a1da887a0:44421 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T03:45:16,830 INFO [RS:0;237a1da887a0:44421 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T03:45:16,830 DEBUG [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,44421,1732333516396 2024-11-23T03:45:16,830 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [237a1da887a0,44421,1732333516396] 2024-11-23T03:45:16,831 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [237a1da887a0,35247,1732333516428] 2024-11-23T03:45:16,835 INFO [RS:1;237a1da887a0:35247 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T03:45:16,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T03:45:16,837 INFO [RS:1;237a1da887a0:35247 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T03:45:16,837 INFO [RS:0;237a1da887a0:44421 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T03:45:16,840 INFO [RS:0;237a1da887a0:44421 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T03:45:16,840 INFO [RS:1;237a1da887a0:35247 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T03:45:16,841 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,841 INFO [RS:0;237a1da887a0:44421 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T03:45:16,841 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,844 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T03:45:16,844 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T03:45:16,845 INFO [RS:0;237a1da887a0:44421 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T03:45:16,845 INFO [RS:1;237a1da887a0:35247 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T03:45:16,845 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,845 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,845 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,845 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/237a1da887a0:0, corePoolSize=2, maxPoolSize=2 2024-11-23T03:45:16,846 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/237a1da887a0:0, corePoolSize=2, maxPoolSize=2 2024-11-23T03:45:16,846 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,846 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,847 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,847 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,847 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,847 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:16,847 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/237a1da887a0:0, corePoolSize=3, maxPoolSize=3 2024-11-23T03:45:16,847 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/237a1da887a0:0, corePoolSize=3, maxPoolSize=3 2024-11-23T03:45:16,847 DEBUG [RS:0;237a1da887a0:44421 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/237a1da887a0:0, corePoolSize=3, maxPoolSize=3 2024-11-23T03:45:16,847 DEBUG [RS:1;237a1da887a0:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/237a1da887a0:0, corePoolSize=3, maxPoolSize=3 2024-11-23T03:45:16,849 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,849 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,849 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,849 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,849 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,849 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,849 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,849 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,849 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,849 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,44421,1732333516396-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T03:45:16,849 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,849 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,35247,1732333516428-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T03:45:16,850 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T03:45:16,850 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:16,850 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:16,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T03:45:16,862 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T03:45:16,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:16,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:16,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T03:45:16,868 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T03:45:16,868 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:16,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:16,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T03:45:16,872 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T03:45:16,872 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,35247,1732333516428-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,872 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,872 INFO [RS:1;237a1da887a0:35247 {}] regionserver.Replication(171): 237a1da887a0,35247,1732333516428 started 2024-11-23T03:45:16,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T03:45:16,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:16,874 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T03:45:16,874 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,44421,1732333516396-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:16,874 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,874 INFO [RS:0;237a1da887a0:44421 {}] regionserver.Replication(171): 237a1da887a0,44421,1732333516396 started 2024-11-23T03:45:16,874 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T03:45:16,875 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740 2024-11-23T03:45:16,875 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740 2024-11-23T03:45:16,875 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T03:45:16,875 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T03:45:16,875 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T03:45:16,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T03:45:16,888 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,889 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(1482): Serving as 237a1da887a0,35247,1732333516428, RpcServer on 237a1da887a0/172.17.0.2:35247, sessionid=0x10142eda0ac0002 2024-11-23T03:45:16,889 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T03:45:16,889 DEBUG [RS:1;237a1da887a0:35247 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 237a1da887a0,35247,1732333516428 2024-11-23T03:45:16,889 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '237a1da887a0,35247,1732333516428' 2024-11-23T03:45:16,889 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T03:45:16,890 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T03:45:16,891 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T03:45:16,891 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T03:45:16,891 DEBUG [RS:1;237a1da887a0:35247 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 237a1da887a0,35247,1732333516428 2024-11-23T03:45:16,891 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '237a1da887a0,35247,1732333516428' 2024-11-23T03:45:16,891 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T03:45:16,891 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T03:45:16,892 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71857764, jitterRate=0.07076412439346313}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T03:45:16,892 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T03:45:16,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732333516829Initializing all the Stores at 1732333516829Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333516829Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333516835 (+6 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333516835Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333516835Cleaning up temporary data from old regions at 1732333516875 (+40 ms)Region opened successfully at 1732333516893 (+18 ms) 2024-11-23T03:45:16,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T03:45:16,893 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T03:45:16,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T03:45:16,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T03:45:16,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T03:45:16,894 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T03:45:16,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732333516893Disabling compacts and flushes for region at 1732333516893Disabling writes for close at 1732333516893Writing region close event to WAL at 1732333516894 (+1 ms)Closed at 1732333516894 2024-11-23T03:45:16,894 DEBUG [RS:1;237a1da887a0:35247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T03:45:16,894 INFO [RS:1;237a1da887a0:35247 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T03:45:16,894 INFO [RS:1;237a1da887a0:35247 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T03:45:16,894 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:16,894 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(1482): Serving as 237a1da887a0,44421,1732333516396, RpcServer on 237a1da887a0/172.17.0.2:44421, sessionid=0x10142eda0ac0001 2024-11-23T03:45:16,895 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T03:45:16,895 DEBUG [RS:0;237a1da887a0:44421 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 237a1da887a0,44421,1732333516396 2024-11-23T03:45:16,895 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '237a1da887a0,44421,1732333516396' 2024-11-23T03:45:16,895 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T03:45:16,895 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T03:45:16,895 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T03:45:16,896 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T03:45:16,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T03:45:16,896 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T03:45:16,896 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T03:45:16,896 DEBUG [RS:0;237a1da887a0:44421 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 237a1da887a0,44421,1732333516396 2024-11-23T03:45:16,897 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '237a1da887a0,44421,1732333516396' 2024-11-23T03:45:16,897 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T03:45:16,897 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T03:45:16,897 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T03:45:16,898 DEBUG [RS:0;237a1da887a0:44421 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T03:45:16,898 INFO [RS:0;237a1da887a0:44421 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T03:45:16,898 INFO [RS:0;237a1da887a0:44421 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T03:45:16,898 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T03:45:16,995 INFO [RS:1;237a1da887a0:35247 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T03:45:16,996 INFO [RS:1;237a1da887a0:35247 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=237a1da887a0%2C35247%2C1732333516428, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,35247,1732333516428, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs, maxLogs=55 2024-11-23T03:45:16,998 INFO [RS:0;237a1da887a0:44421 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T03:45:16,999 INFO [RS:0;237a1da887a0:44421 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=237a1da887a0%2C44421%2C1732333516396, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,44421,1732333516396, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs, maxLogs=55 2024-11-23T03:45:17,018 INFO [RS:0;237a1da887a0:44421 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,44421,1732333516396/237a1da887a0%2C44421%2C1732333516396.1732333517000 2024-11-23T03:45:17,018 INFO [RS:1;237a1da887a0:35247 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,35247,1732333516428/237a1da887a0%2C35247%2C1732333516428.1732333516998 2024-11-23T03:45:17,018 DEBUG [RS:0;237a1da887a0:44421 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T03:45:17,018 DEBUG [RS:1;237a1da887a0:35247 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T03:45:17,049 DEBUG [237a1da887a0:39357 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-23T03:45:17,049 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(204): Hosts are {237a1da887a0=0} racks are {/default-rack=0} 2024-11-23T03:45:17,050 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T03:45:17,050 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T03:45:17,050 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T03:45:17,050 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T03:45:17,050 INFO [237a1da887a0:39357 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T03:45:17,050 INFO [237a1da887a0:39357 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T03:45:17,050 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T03:45:17,050 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=237a1da887a0,35247,1732333516428 2024-11-23T03:45:17,052 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 237a1da887a0,35247,1732333516428, state=OPENING 2024-11-23T03:45:17,054 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T03:45:17,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:17,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:17,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:17,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:17,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:17,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:17,058 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T03:45:17,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=237a1da887a0,35247,1732333516428}] 2024-11-23T03:45:17,212 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T03:45:17,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32799, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T03:45:17,217 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T03:45:17,217 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T03:45:17,218 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T03:45:17,219 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=237a1da887a0%2C35247%2C1732333516428.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,35247,1732333516428, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs, maxLogs=55 2024-11-23T03:45:17,247 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,35247,1732333516428/237a1da887a0%2C35247%2C1732333516428.meta.1732333517220.meta 2024-11-23T03:45:17,247 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T03:45:17,248 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T03:45:17,248 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T03:45:17,248 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T03:45:17,249 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T03:45:17,249 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T03:45:17,249 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:17,249 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T03:45:17,249 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T03:45:17,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T03:45:17,251 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T03:45:17,252 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:17,252 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:17,252 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T03:45:17,253 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T03:45:17,253 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:17,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:17,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T03:45:17,254 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T03:45:17,254 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:17,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:17,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T03:45:17,255 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T03:45:17,255 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:17,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:17,256 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T03:45:17,257 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740 2024-11-23T03:45:17,257 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740 2024-11-23T03:45:17,258 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T03:45:17,258 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T03:45:17,258 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T03:45:17,259 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T03:45:17,260 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67134501, jitterRate=3.8202106952667236E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T03:45:17,260 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T03:45:17,261 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732333517249Writing region info on filesystem at 1732333517249Initializing all the Stores at 1732333517250 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333517250Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333517250Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333517250Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333517250Cleaning up temporary data from old regions at 1732333517258 (+8 ms)Running coprocessor post-open hooks at 1732333517260 (+2 ms)Region opened successfully at 1732333517261 (+1 ms) 2024-11-23T03:45:17,262 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732333517211 2024-11-23T03:45:17,266 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T03:45:17,266 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T03:45:17,266 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=237a1da887a0,35247,1732333516428 2024-11-23T03:45:17,267 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 237a1da887a0,35247,1732333516428, state=OPEN 2024-11-23T03:45:17,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T03:45:17,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T03:45:17,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T03:45:17,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:17,271 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=237a1da887a0,35247,1732333516428 2024-11-23T03:45:17,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:17,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:17,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T03:45:17,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=237a1da887a0,35247,1732333516428 in 213 msec 2024-11-23T03:45:17,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T03:45:17,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 379 msec 2024-11-23T03:45:17,277 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T03:45:17,278 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T03:45:17,282 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T03:45:17,282 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=237a1da887a0,35247,1732333516428, seqNum=-1] 2024-11-23T03:45:17,282 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T03:45:17,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60685, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T03:45:17,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 557 msec 2024-11-23T03:45:17,291 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732333517291, completionTime=-1 2024-11-23T03:45:17,291 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-23T03:45:17,291 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T03:45:17,294 INFO [master/237a1da887a0:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-23T03:45:17,294 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732333577294 2024-11-23T03:45:17,294 INFO [master/237a1da887a0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732333637294 2024-11-23T03:45:17,294 INFO [master/237a1da887a0:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-23T03:45:17,294 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,39357,1732333516336-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:17,294 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,39357,1732333516336-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:17,294 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,39357,1732333516336-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:17,295 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-237a1da887a0:39357, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:17,295 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:17,295 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:17,298 DEBUG [master/237a1da887a0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T03:45:17,301 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.828sec 2024-11-23T03:45:17,301 INFO [master/237a1da887a0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T03:45:17,301 INFO [master/237a1da887a0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T03:45:17,301 INFO [master/237a1da887a0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T03:45:17,301 INFO [master/237a1da887a0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T03:45:17,302 INFO [master/237a1da887a0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T03:45:17,302 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,39357,1732333516336-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T03:45:17,302 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,39357,1732333516336-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T03:45:17,303 DEBUG [master/237a1da887a0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T03:45:17,303 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T03:45:17,303 INFO [master/237a1da887a0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,39357,1732333516336-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:17,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T03:45:17,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 237a1da887a0,39357,-1 for getting cluster id 2024-11-23T03:45:17,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T03:45:17,369 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2c75c6a7-08bf-43b5-a5b3-05b5621cd0c5' 2024-11-23T03:45:17,370 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T03:45:17,370 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2c75c6a7-08bf-43b5-a5b3-05b5621cd0c5" 2024-11-23T03:45:17,370 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae3ec1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T03:45:17,370 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [237a1da887a0,39357,-1] 2024-11-23T03:45:17,371 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T03:45:17,375 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:17,377 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T03:45:17,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T03:45:17,379 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T03:45:17,381 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=237a1da887a0,35247,1732333516428, seqNum=-1] 2024-11-23T03:45:17,381 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T03:45:17,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T03:45:17,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=237a1da887a0,39357,1732333516336 2024-11-23T03:45:17,387 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T03:45:17,389 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 237a1da887a0,39357,1732333516336 2024-11-23T03:45:17,389 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23035661 2024-11-23T03:45:17,389 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T03:45:17,392 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T03:45:17,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39357 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732333517387', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T03:45:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39357 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732333517387 2024-11-23T03:45:17,395 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732333517387 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T03:45:17,396 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:17,396 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39357 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732333517387" procId is: 4 2024-11-23T03:45:17,397 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732333517387 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T03:45:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T03:45:17,419 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732333517387-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4dbfd6cc71ec7598dc81c3407987b732, NAME => 'testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732333517387', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd 2024-11-23T03:45:17,427 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732333517387-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:17,428 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732333517387-pool-0 {}] regionserver.HRegion(1722): Closing 4dbfd6cc71ec7598dc81c3407987b732, disabling compactions & flushes 2024-11-23T03:45:17,428 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732333517387-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:17,428 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732333517387-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:17,428 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732333517387-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. after waiting 0 ms 2024-11-23T03:45:17,428 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732333517387-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:17,428 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732333517387-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:17,428 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732333517387-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4dbfd6cc71ec7598dc81c3407987b732: Waiting for close lock at 1732333517427Disabling compacts and flushes for region at 1732333517427Disabling writes for close at 1732333517428 (+1 ms)Writing region close event to WAL at 1732333517428Closed at 1732333517428 2024-11-23T03:45:17,429 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732333517387 execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T03:45:17,430 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732333517429"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732333517429"}]},"ts":"1732333517429"} 2024-11-23T03:45:17,434 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T03:45:17,435 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732333517387 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T03:45:17,436 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732333517387","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732333517435"}]},"ts":"1732333517435"} 2024-11-23T03:45:17,441 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732333517387, state=ENABLING in hbase:meta 2024-11-23T03:45:17,441 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {237a1da887a0=0} racks are {/default-rack=0} 2024-11-23T03:45:17,442 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T03:45:17,442 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T03:45:17,442 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T03:45:17,442 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T03:45:17,442 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T03:45:17,442 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T03:45:17,442 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T03:45:17,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732333517387, region=4dbfd6cc71ec7598dc81c3407987b732, ASSIGN}] 2024-11-23T03:45:17,444 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732333517387, region=4dbfd6cc71ec7598dc81c3407987b732, ASSIGN 2024-11-23T03:45:17,445 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732333517387, region=4dbfd6cc71ec7598dc81c3407987b732, ASSIGN; state=OFFLINE, location=237a1da887a0,35247,1732333516428; forceNewPlan=false, retain=false 2024-11-23T03:45:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T03:45:17,596 INFO [237a1da887a0:39357 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T03:45:17,596 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4dbfd6cc71ec7598dc81c3407987b732, regionState=OPENING, regionLocation=237a1da887a0,35247,1732333516428 2024-11-23T03:45:17,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732333517387, region=4dbfd6cc71ec7598dc81c3407987b732, ASSIGN because future has completed 2024-11-23T03:45:17,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4dbfd6cc71ec7598dc81c3407987b732, server=237a1da887a0,35247,1732333516428}] 2024-11-23T03:45:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T03:45:17,755 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:17,756 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4dbfd6cc71ec7598dc81c3407987b732, NAME => 'testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732.', STARTKEY => '', ENDKEY => ''} 2024-11-23T03:45:17,756 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732333517387 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:17,756 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:17,756 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:17,756 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:17,757 INFO [StoreOpener-4dbfd6cc71ec7598dc81c3407987b732-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:17,761 INFO [StoreOpener-4dbfd6cc71ec7598dc81c3407987b732-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dbfd6cc71ec7598dc81c3407987b732 columnFamilyName family 2024-11-23T03:45:17,761 DEBUG [StoreOpener-4dbfd6cc71ec7598dc81c3407987b732-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:17,762 INFO [StoreOpener-4dbfd6cc71ec7598dc81c3407987b732-1 {}] regionserver.HStore(327): Store=4dbfd6cc71ec7598dc81c3407987b732/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T03:45:17,762 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:17,763 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:17,763 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:17,763 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:17,763 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:17,764 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:17,778 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T03:45:17,778 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4dbfd6cc71ec7598dc81c3407987b732; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69806016, jitterRate=0.040190696716308594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T03:45:17,778 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:17,779 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4dbfd6cc71ec7598dc81c3407987b732: Running coprocessor pre-open hook at 1732333517757Writing region info on filesystem at 1732333517757Initializing all the Stores at 1732333517757Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333517757Cleaning up temporary data from old regions at 1732333517763 (+6 ms)Running coprocessor post-open hooks at 1732333517778 (+15 ms)Region opened successfully at 1732333517779 (+1 ms) 2024-11-23T03:45:17,780 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732., pid=6, masterSystemTime=1732333517753 2024-11-23T03:45:17,785 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:17,785 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:17,785 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4dbfd6cc71ec7598dc81c3407987b732, regionState=OPEN, openSeqNum=2, regionLocation=237a1da887a0,35247,1732333516428 2024-11-23T03:45:17,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4dbfd6cc71ec7598dc81c3407987b732, server=237a1da887a0,35247,1732333516428 because future has completed 2024-11-23T03:45:17,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T03:45:17,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4dbfd6cc71ec7598dc81c3407987b732, server=237a1da887a0,35247,1732333516428 in 193 msec 2024-11-23T03:45:17,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T03:45:17,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732333517387, region=4dbfd6cc71ec7598dc81c3407987b732, ASSIGN in 354 msec 2024-11-23T03:45:17,801 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732333517387 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T03:45:17,802 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732333517387","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732333517801"}]},"ts":"1732333517801"} 2024-11-23T03:45:17,805 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732333517387, state=ENABLED in hbase:meta 2024-11-23T03:45:17,806 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732333517387 execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T03:45:17,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732333517387 in 413 msec 2024-11-23T03:45:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T03:45:18,030 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732333517387 completed 2024-11-23T03:45:18,030 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732333517387 get assigned. Timeout = 60000ms 2024-11-23T03:45:18,031 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T03:45:18,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732333517387 assigned to meta. Checking AM states. 2024-11-23T03:45:18,034 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T03:45:18,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732333517387 assigned. 2024-11-23T03:45:18,038 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732333517387', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732., hostname=237a1da887a0,35247,1732333516428, seqNum=2] 2024-11-23T03:45:18,043 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4dbfd6cc71ec7598dc81c3407987b732 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-23T03:45:18,053 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 27d97eee5ba74798b57488971c40b547 2024-11-23T03:45:18,067 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/.tmp/family/27d97eee5ba74798b57488971c40b547 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-23T03:45:18,067 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/.tmp/family/27d97eee5ba74798b57488971c40b547 2024-11-23T03:45:18,070 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/.tmp/family/27d97eee5ba74798b57488971c40b547 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/family/27d97eee5ba74798b57488971c40b547 2024-11-23T03:45:18,071 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/family/27d97eee5ba74798b57488971c40b547, delay=957 ms 2024-11-23T03:45:18,072 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/family/27d97eee5ba74798b57488971c40b547, entries=2, sequenceid=6, filesize=4.9 K 2024-11-23T03:45:18,073 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 4dbfd6cc71ec7598dc81c3407987b732 in 30ms, sequenceid=6, compaction requested=false 2024-11-23T03:45:18,073 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732333517387' 2024-11-23T03:45:18,074 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4dbfd6cc71ec7598dc81c3407987b732: 2024-11-23T03:45:18,542 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T03:45:19,029 DEBUG [hfile-prefetch-1732333515725 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 27d97eee5ba74798b57488971c40b547_0. 2024-11-23T03:45:19,030 DEBUG [hfile-prefetch-1732333515725 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 27d97eee5ba74798b57488971c40b547_117. 2024-11-23T03:45:19,030 DEBUG [hfile-prefetch-1732333515725 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/family/27d97eee5ba74798b57488971c40b547, with total blocks 2, and data blocks 1 2024-11-23T03:45:19,030 DEBUG [hfile-prefetch-1732333515725 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-23T03:45:19,031 DEBUG [hfile-prefetch-1732333515725 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/family/27d97eee5ba74798b57488971c40b547, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-23T03:45:19,031 DEBUG [hfile-prefetch-1732333515725 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/family/27d97eee5ba74798b57488971c40b547. Will sleep 100ms and try the verification again. 2024-11-23T03:45:19,041 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T03:45:19,041 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T03:45:19,041 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T03:45:19,041 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T03:45:19,048 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-23T03:45:19,048 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732333519041 2024-11-23T03:45:19,048 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T03:45:19,074 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;237a1da887a0:35247,5,FailOnTimeoutGroup] 2024-11-23T03:45:19,074 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '237a1da887a0,35247,1732333516428' ***** 2024-11-23T03:45:19,074 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-23T03:45:19,075 INFO [RS:1;237a1da887a0:35247 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T03:45:19,075 INFO [RS:1;237a1da887a0:35247 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.persistence 2024-11-23T03:45:19,075 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T03:45:19,075 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T03:45:19,075 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T03:45:19,075 INFO [RS:1;237a1da887a0:35247 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T03:45:19,075 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T03:45:19,085 DEBUG [RS:1;237a1da887a0:35247 {}] bucket.BucketCache(1487): Thread RS:1;237a1da887a0:35247 started persisting bucket cache to file 2024-11-23T03:45:19,085 DEBUG [RS:1;237a1da887a0:35247 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T03:45:19,085 DEBUG [RS:1;237a1da887a0:35247 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T03:45:19,088 DEBUG [RS:1;237a1da887a0:35247 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-23T03:45:19,089 DEBUG [RS:1;237a1da887a0:35247 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732333519085 2024-11-23T03:45:19,089 DEBUG [RS:1;237a1da887a0:35247 {}] bucket.BucketCache(1509): Thread RS:1;237a1da887a0:35247 finished persisting bucket cache to file, renaming 2024-11-23T03:45:19,089 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T03:45:19,089 INFO [RS:1;237a1da887a0:35247 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T03:45:19,089 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T03:45:19,089 INFO [RS:1;237a1da887a0:35247 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T03:45:19,089 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(3091): Received CLOSE for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:19,089 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(959): stopping server 237a1da887a0,35247,1732333516428 2024-11-23T03:45:19,089 INFO [RS:1;237a1da887a0:35247 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T03:45:19,090 INFO [RS:1;237a1da887a0:35247 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;237a1da887a0:35247. 2024-11-23T03:45:19,090 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T03:45:19,090 DEBUG [RS:1;237a1da887a0:35247 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T03:45:19,090 DEBUG [RS:1;237a1da887a0:35247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:19,090 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4dbfd6cc71ec7598dc81c3407987b732, disabling compactions & flushes 2024-11-23T03:45:19,090 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:19,090 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:19,090 INFO [RS:1;237a1da887a0:35247 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T03:45:19,090 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. after waiting 0 ms 2024-11-23T03:45:19,090 INFO [RS:1;237a1da887a0:35247 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T03:45:19,090 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:19,090 INFO [RS:1;237a1da887a0:35247 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T03:45:19,090 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T03:45:19,090 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T03:45:19,090 DEBUG [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(1325): Online Regions={4dbfd6cc71ec7598dc81c3407987b732=testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732., 1588230740=hbase:meta,,1.1588230740} 2024-11-23T03:45:19,090 DEBUG [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:19,090 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T03:45:19,091 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T03:45:19,091 DEBUG [StoreFileCloser-4dbfd6cc71ec7598dc81c3407987b732-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/family/27d97eee5ba74798b57488971c40b547 2024-11-23T03:45:19,091 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T03:45:19,091 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T03:45:19,091 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T03:45:19,091 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T03:45:19,091 WARN [hfile-prefetch-1732333515725 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/family/27d97eee5ba74798b57488971c40b547, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-23T03:45:19,091 DEBUG [hfile-prefetch-1732333515725 {}] hfile.PrefetchExecutor(121): Prefetch completed for 27d97eee5ba74798b57488971c40b547 2024-11-23T03:45:19,091 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-23T03:45:19,095 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-23T03:45:19,095 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:19,096 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4dbfd6cc71ec7598dc81c3407987b732: Waiting for close lock at 1732333519090Running coprocessor pre-close hooks at 1732333519090Disabling compacts and flushes for region at 1732333519090Disabling writes for close at 1732333519090Writing region close event to WAL at 1732333519091 (+1 ms)Running coprocessor post-close hooks at 1732333519095 (+4 ms)Closed at 1732333519095 2024-11-23T03:45:19,096 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:19,102 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3a7b5eee8d7745eb9558f4ece26764e1 2024-11-23T03:45:19,116 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/info/3a7b5eee8d7745eb9558f4ece26764e1 is 209, key is testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732./info:regioninfo/1732333517785/Put/seqid=0 2024-11-23T03:45:19,116 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/info/3a7b5eee8d7745eb9558f4ece26764e1 2024-11-23T03:45:19,125 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a8575720a67848ee81fe3ccae9e4f04d 2024-11-23T03:45:19,140 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/ns/a8575720a67848ee81fe3ccae9e4f04d is 43, key is default/ns:d/1732333517285/Put/seqid=0 2024-11-23T03:45:19,140 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/ns/a8575720a67848ee81fe3ccae9e4f04d 2024-11-23T03:45:19,149 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 56dae1eac8184e578e886de68acba834 2024-11-23T03:45:19,156 INFO [regionserver/237a1da887a0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T03:45:19,163 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/table/56dae1eac8184e578e886de68acba834 is 80, key is testBlockEvictionOnGracefulStop1732333517387/table:state/1732333517801/Put/seqid=0 2024-11-23T03:45:19,164 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/table/56dae1eac8184e578e886de68acba834 2024-11-23T03:45:19,167 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/info/3a7b5eee8d7745eb9558f4ece26764e1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/3a7b5eee8d7745eb9558f4ece26764e1 2024-11-23T03:45:19,169 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/3a7b5eee8d7745eb9558f4ece26764e1, delay=919 ms 2024-11-23T03:45:19,170 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/3a7b5eee8d7745eb9558f4ece26764e1, entries=10, sequenceid=11, filesize=7.0 K 2024-11-23T03:45:19,170 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/ns/a8575720a67848ee81fe3ccae9e4f04d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/ns/a8575720a67848ee81fe3ccae9e4f04d 2024-11-23T03:45:19,171 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/ns/a8575720a67848ee81fe3ccae9e4f04d, delay=995 ms 2024-11-23T03:45:19,172 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/ns/a8575720a67848ee81fe3ccae9e4f04d, entries=2, sequenceid=11, filesize=5.0 K 2024-11-23T03:45:19,173 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/table/56dae1eac8184e578e886de68acba834 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/table/56dae1eac8184e578e886de68acba834 2024-11-23T03:45:19,174 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/table/56dae1eac8184e578e886de68acba834, delay=911 ms 2024-11-23T03:45:19,175 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/table/56dae1eac8184e578e886de68acba834, entries=2, sequenceid=11, filesize=5.3 K 2024-11-23T03:45:19,176 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=11, compaction requested=false 2024-11-23T03:45:19,176 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/3a7b5eee8d7745eb9558f4ece26764e1 2024-11-23T03:45:19,177 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/ns/a8575720a67848ee81fe3ccae9e4f04d 2024-11-23T03:45:19,178 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/table/56dae1eac8184e578e886de68acba834 2024-11-23T03:45:19,182 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-23T03:45:19,182 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T03:45:19,182 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T03:45:19,183 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732333519090Running coprocessor pre-close hooks at 1732333519090Disabling compacts and flushes for region at 1732333519090Disabling writes for close at 1732333519091 (+1 ms)Obtaining lock to block concurrent updates at 1732333519091Preparing flush snapshotting stores in 1588230740 at 1732333519091Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732333519092 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732333519092Flushing 1588230740/info: creating writer at 1732333519092Flushing 1588230740/info: appending metadata at 1732333519115 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732333519115Flushing 1588230740/ns: creating writer at 1732333519119 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732333519139 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732333519139Flushing 1588230740/table: creating writer at 1732333519143 (+4 ms)Flushing 1588230740/table: appending metadata at 1732333519163 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732333519163Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59d31c0c: reopening flushed file at 1732333519167 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@293f94ed: reopening flushed file at 1732333519170 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79f15b6f: reopening flushed file at 1732333519173 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=11, compaction requested=false at 1732333519176 (+3 ms)Writing region close event to WAL at 1732333519178 (+2 ms)Running coprocessor post-close hooks at 1732333519182 (+4 ms)Closed at 1732333519182 2024-11-23T03:45:19,183 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T03:45:19,291 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(976): stopping server 237a1da887a0,35247,1732333516428; all regions closed. 2024-11-23T03:45:19,293 DEBUG [RS:1;237a1da887a0:35247 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs 2024-11-23T03:45:19,293 INFO [RS:1;237a1da887a0:35247 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 237a1da887a0%2C35247%2C1732333516428.meta:.meta(num 1732333517220) 2024-11-23T03:45:19,295 DEBUG [RS:1;237a1da887a0:35247 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs 2024-11-23T03:45:19,295 INFO [RS:1;237a1da887a0:35247 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 237a1da887a0%2C35247%2C1732333516428:(num 1732333516998) 2024-11-23T03:45:19,295 DEBUG [RS:1;237a1da887a0:35247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:19,295 INFO [RS:1;237a1da887a0:35247 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T03:45:19,295 INFO [RS:1;237a1da887a0:35247 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T03:45:19,295 INFO [RS:1;237a1da887a0:35247 {}] hbase.ChoreService(370): Chore service for: regionserver/237a1da887a0:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T03:45:19,296 INFO [RS:1;237a1da887a0:35247 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T03:45:19,296 INFO [regionserver/237a1da887a0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T03:45:19,296 INFO [RS:1;237a1da887a0:35247 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35247 2024-11-23T03:45:19,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/237a1da887a0,35247,1732333516428 2024-11-23T03:45:19,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T03:45:19,301 INFO [RS:1;237a1da887a0:35247 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T03:45:19,302 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [237a1da887a0,35247,1732333516428] 2024-11-23T03:45:19,303 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/237a1da887a0,35247,1732333516428 already deleted, retry=false 2024-11-23T03:45:19,304 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 237a1da887a0,35247,1732333516428 on 237a1da887a0,39357,1732333516336 2024-11-23T03:45:19,309 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 237a1da887a0,35247,1732333516428, splitWal=true, meta=true 2024-11-23T03:45:19,311 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 237a1da887a0,35247,1732333516428 (carryingMeta=true) 237a1da887a0,35247,1732333516428/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1704c18b[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-23T03:45:19,312 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 237a1da887a0,35247,1732333516428, splitWal=true, meta=true 2024-11-23T03:45:19,312 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 237a1da887a0,35247,1732333516428, splitWal=true, meta=true, isMeta: true 2024-11-23T03:45:19,313 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,35247,1732333516428-splitting 2024-11-23T03:45:19,313 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,35247,1732333516428-splitting dir is empty, no logs to split. 2024-11-23T03:45:19,313 INFO [PEWorker-2 {}] master.SplitWALManager(105): 237a1da887a0,35247,1732333516428 WAL count=0, meta=true 2024-11-23T03:45:19,315 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,35247,1732333516428-splitting dir is empty, no logs to split. 2024-11-23T03:45:19,315 INFO [PEWorker-2 {}] master.SplitWALManager(105): 237a1da887a0,35247,1732333516428 WAL count=0, meta=true 2024-11-23T03:45:19,315 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 237a1da887a0,35247,1732333516428 WAL splitting is done? wals=0, meta=true 2024-11-23T03:45:19,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T03:45:19,318 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T03:45:19,319 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T03:45:19,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:19,403 INFO [RS:1;237a1da887a0:35247 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T03:45:19,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x10142eda0ac0002, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:19,403 INFO [RS:1;237a1da887a0:35247 {}] regionserver.HRegionServer(1031): Exiting; stopping=237a1da887a0,35247,1732333516428; zookeeper connection closed. 2024-11-23T03:45:19,403 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@52e1578 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@52e1578 2024-11-23T03:45:19,469 DEBUG [237a1da887a0:39357 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T03:45:19,470 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=237a1da887a0,44421,1732333516396 2024-11-23T03:45:19,471 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 237a1da887a0,44421,1732333516396, state=OPENING 2024-11-23T03:45:19,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T03:45:19,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T03:45:19,474 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T03:45:19,474 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:19,474 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:19,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=237a1da887a0,44421,1732333516396}] 2024-11-23T03:45:19,598 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/237a1da887a0:0 server-side Connection retries=45 2024-11-23T03:45:19,598 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:19,598 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:19,598 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T03:45:19,598 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T03:45:19,598 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T03:45:19,598 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T03:45:19,599 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T03:45:19,600 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39237 2024-11-23T03:45:19,602 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39237 connecting to ZooKeeper ensemble=127.0.0.1:49805 2024-11-23T03:45:19,602 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:19,602 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:19,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:392370x0, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T03:45:19,607 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39237-0x10142eda0ac0003 connected 2024-11-23T03:45:19,607 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-23T03:45:19,607 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39237-0x10142eda0ac0003, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-23T03:45:19,608 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T03:45:19,612 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.cache 2024-11-23T03:45:19,612 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T03:45:19,612 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T03:45:19,613 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T03:45:19,614 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T03:45:19,615 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T03:45:19,615 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-23T03:45:19,616 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T03:45:19,616 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39237-0x10142eda0ac0003, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T03:45:19,617 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-23T03:45:19,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39237-0x10142eda0ac0003, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T03:45:19,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39237 2024-11-23T03:45:19,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39237 2024-11-23T03:45:19,620 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-23T03:45:19,621 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T03:45:19,621 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-23T03:45:19,621 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.persistence is complete. 2024-11-23T03:45:19,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39237 2024-11-23T03:45:19,627 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T03:45:19,627 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39237 2024-11-23T03:45:19,627 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39237 2024-11-23T03:45:19,629 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52283, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T03:45:19,630 INFO [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer(746): ClusterId : 2c75c6a7-08bf-43b5-a5b3-05b5621cd0c5 2024-11-23T03:45:19,630 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T03:45:19,632 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T03:45:19,633 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T03:45:19,635 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T03:45:19,636 DEBUG [RS:2;237a1da887a0:39237 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1112ed65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=237a1da887a0/172.17.0.2:0 2024-11-23T03:45:19,636 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T03:45:19,636 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T03:45:19,637 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T03:45:19,638 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=237a1da887a0%2C44421%2C1732333516396.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,44421,1732333516396, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs, maxLogs=55 2024-11-23T03:45:19,650 DEBUG [RS:2;237a1da887a0:39237 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;237a1da887a0:39237 2024-11-23T03:45:19,650 INFO [RS:2;237a1da887a0:39237 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T03:45:19,651 INFO [RS:2;237a1da887a0:39237 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T03:45:19,651 DEBUG [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T03:45:19,652 INFO [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer(2659): reportForDuty to master=237a1da887a0,39357,1732333516336 with port=39237, startcode=1732333519597 2024-11-23T03:45:19,652 DEBUG [RS:2;237a1da887a0:39237 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T03:45:19,654 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57575, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T03:45:19,655 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 237a1da887a0,39237,1732333519597 2024-11-23T03:45:19,655 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39357 {}] master.ServerManager(517): Registering regionserver=237a1da887a0,39237,1732333519597 2024-11-23T03:45:19,656 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,44421,1732333516396/237a1da887a0%2C44421%2C1732333516396.meta.1732333519638.meta 2024-11-23T03:45:19,657 DEBUG [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd 2024-11-23T03:45:19,657 DEBUG [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T03:45:19,657 DEBUG [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T03:45:19,657 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T03:45:19,657 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T03:45:19,658 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T03:45:19,658 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T03:45:19,658 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T03:45:19,658 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T03:45:19,658 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:19,658 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T03:45:19,658 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T03:45:19,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T03:45:19,660 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T03:45:19,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T03:45:19,660 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:19,661 DEBUG [RS:2;237a1da887a0:39237 {}] zookeeper.ZKUtil(111): regionserver:39237-0x10142eda0ac0003, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/237a1da887a0,39237,1732333519597 2024-11-23T03:45:19,661 WARN [RS:2;237a1da887a0:39237 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T03:45:19,661 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [237a1da887a0,39237,1732333519597] 2024-11-23T03:45:19,661 INFO [RS:2;237a1da887a0:39237 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T03:45:19,661 DEBUG [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,39237,1732333519597 2024-11-23T03:45:19,664 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/3a7b5eee8d7745eb9558f4ece26764e1, delay=943 ms 2024-11-23T03:45:19,665 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/3a7b5eee8d7745eb9558f4ece26764e1 2024-11-23T03:45:19,666 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:19,666 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T03:45:19,666 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T03:45:19,666 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:19,671 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/ns/a8575720a67848ee81fe3ccae9e4f04d, delay=909 ms 2024-11-23T03:45:19,671 INFO [RS:2;237a1da887a0:39237 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T03:45:19,672 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/ns/a8575720a67848ee81fe3ccae9e4f04d 2024-11-23T03:45:19,672 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:19,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T03:45:19,673 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T03:45:19,673 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:19,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:19,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T03:45:19,674 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T03:45:19,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:19,676 INFO [RS:2;237a1da887a0:39237 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T03:45:19,678 INFO [RS:2;237a1da887a0:39237 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T03:45:19,678 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:19,679 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/table/56dae1eac8184e578e886de68acba834, delay=906 ms 2024-11-23T03:45:19,680 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/table/56dae1eac8184e578e886de68acba834 2024-11-23T03:45:19,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T03:45:19,680 INFO [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T03:45:19,681 INFO [RS:2;237a1da887a0:39237 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T03:45:19,681 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:19,681 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:19,681 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:19,681 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:19,681 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:19,681 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:19,681 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/237a1da887a0:0, corePoolSize=2, maxPoolSize=2 2024-11-23T03:45:19,681 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:19,681 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:19,681 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:19,682 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:19,682 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:19,682 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/237a1da887a0:0, corePoolSize=1, maxPoolSize=1 2024-11-23T03:45:19,682 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/237a1da887a0:0, corePoolSize=3, maxPoolSize=3 2024-11-23T03:45:19,682 DEBUG [RS:2;237a1da887a0:39237 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/237a1da887a0:0, corePoolSize=3, maxPoolSize=3 2024-11-23T03:45:19,682 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T03:45:19,682 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740 2024-11-23T03:45:19,683 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740 2024-11-23T03:45:19,683 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T03:45:19,683 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T03:45:19,683 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:19,684 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:19,684 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:19,684 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:19,684 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:19,684 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,39237,1732333519597-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T03:45:19,684 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T03:45:19,685 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T03:45:19,686 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63158522, jitterRate=-0.05886468291282654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T03:45:19,686 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T03:45:19,687 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732333519658Writing region info on filesystem at 1732333519659 (+1 ms)Initializing all the Stores at 1732333519659Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333519659Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333519659Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333519659Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732333519659Cleaning up temporary data from old regions at 1732333519683 (+24 ms)Running coprocessor post-open hooks at 1732333519686 (+3 ms)Region opened successfully at 1732333519687 (+1 ms) 2024-11-23T03:45:19,688 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732333519626 2024-11-23T03:45:19,691 DEBUG [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T03:45:19,691 INFO [RS_OPEN_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T03:45:19,692 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=237a1da887a0,44421,1732333516396 2024-11-23T03:45:19,692 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 237a1da887a0,44421,1732333516396, state=OPEN 2024-11-23T03:45:19,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T03:45:19,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T03:45:19,694 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=237a1da887a0,44421,1732333516396 2024-11-23T03:45:19,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:19,695 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:19,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39237-0x10142eda0ac0003, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T03:45:19,695 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T03:45:19,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-23T03:45:19,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=237a1da887a0,44421,1732333516396 in 220 msec 2024-11-23T03:45:19,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T03:45:19,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 381 msec 2024-11-23T03:45:19,700 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 237a1da887a0,35247,1732333516428 had 2 regions 2024-11-23T03:45:19,700 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 237a1da887a0,35247,1732333516428, splitWal=true, meta=true, isMeta: false 2024-11-23T03:45:19,701 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,35247,1732333516428-splitting dir is empty, no logs to split. 2024-11-23T03:45:19,701 INFO [PEWorker-3 {}] master.SplitWALManager(105): 237a1da887a0,35247,1732333516428 WAL count=0, meta=false 2024-11-23T03:45:19,702 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,35247,1732333516428-splitting dir is empty, no logs to split. 2024-11-23T03:45:19,702 INFO [PEWorker-3 {}] master.SplitWALManager(105): 237a1da887a0,35247,1732333516428 WAL count=0, meta=false 2024-11-23T03:45:19,702 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 237a1da887a0,35247,1732333516428 WAL splitting is done? wals=0, meta=false 2024-11-23T03:45:19,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732333517387, region=4dbfd6cc71ec7598dc81c3407987b732, ASSIGN}] 2024-11-23T03:45:19,704 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732333517387, region=4dbfd6cc71ec7598dc81c3407987b732, ASSIGN 2024-11-23T03:45:19,705 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732333517387, region=4dbfd6cc71ec7598dc81c3407987b732, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T03:45:19,708 INFO [RS:2;237a1da887a0:39237 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T03:45:19,709 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(168): Chore ScheduledChore name=237a1da887a0,39237,1732333519597-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:19,709 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:19,709 INFO [RS:2;237a1da887a0:39237 {}] regionserver.Replication(171): 237a1da887a0,39237,1732333519597 started 2024-11-23T03:45:19,727 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T03:45:19,727 INFO [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer(1482): Serving as 237a1da887a0,39237,1732333519597, RpcServer on 237a1da887a0/172.17.0.2:39237, sessionid=0x10142eda0ac0003 2024-11-23T03:45:19,728 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T03:45:19,728 DEBUG [RS:2;237a1da887a0:39237 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 237a1da887a0,39237,1732333519597 2024-11-23T03:45:19,728 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '237a1da887a0,39237,1732333519597' 2024-11-23T03:45:19,728 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T03:45:19,728 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T03:45:19,729 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T03:45:19,729 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T03:45:19,729 DEBUG [RS:2;237a1da887a0:39237 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 237a1da887a0,39237,1732333519597 2024-11-23T03:45:19,729 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '237a1da887a0,39237,1732333519597' 2024-11-23T03:45:19,729 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T03:45:19,730 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T03:45:19,730 DEBUG [RS:2;237a1da887a0:39237 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T03:45:19,730 INFO [RS:2;237a1da887a0:39237 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T03:45:19,730 INFO [RS:2;237a1da887a0:39237 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T03:45:19,831 INFO [RS:2;237a1da887a0:39237 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T03:45:19,832 INFO [RS:2;237a1da887a0:39237 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=237a1da887a0%2C39237%2C1732333519597, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,39237,1732333519597, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs, maxLogs=55 2024-11-23T03:45:19,850 INFO [RS:2;237a1da887a0:39237 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/WALs/237a1da887a0,39237,1732333519597/237a1da887a0%2C39237%2C1732333519597.1732333519832 2024-11-23T03:45:19,850 DEBUG [RS:2;237a1da887a0:39237 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T03:45:19,855 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(204): Hosts are {237a1da887a0=0} racks are {/default-rack=0} 2024-11-23T03:45:19,856 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T03:45:19,856 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T03:45:19,856 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T03:45:19,856 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T03:45:19,856 INFO [237a1da887a0:39357 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T03:45:19,856 INFO [237a1da887a0:39357 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T03:45:19,856 DEBUG [237a1da887a0:39357 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T03:45:19,856 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4dbfd6cc71ec7598dc81c3407987b732, regionState=OPENING, regionLocation=237a1da887a0,44421,1732333516396 2024-11-23T03:45:19,857 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 237a1da887a0:35247 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 237a1da887a0/172.17.0.2:35247 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T03:45:19,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=237a1da887a0,35247,1732333516428, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=237a1da887a0,35247,1732333516428, seqNum=-1, error=java.net.ConnectException: Call to address=237a1da887a0:35247 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 237a1da887a0/172.17.0.2:35247 2024-11-23T03:45:19,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=237a1da887a0,35247,1732333516428, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-23T03:45:19,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=237a1da887a0,35247,1732333516428, seqNum=-1 from cache 2024-11-23T03:45:19,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 237a1da887a0:35247 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 237a1da887a0/172.17.0.2:35247 2024-11-23T03:45:19,968 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T03:45:19,968 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=237a1da887a0,44421,1732333516396, seqNum=-1] 2024-11-23T03:45:19,969 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T03:45:19,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37955, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T03:45:19,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732333517387, region=4dbfd6cc71ec7598dc81c3407987b732, ASSIGN because future has completed 2024-11-23T03:45:19,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4dbfd6cc71ec7598dc81c3407987b732, server=237a1da887a0,44421,1732333516396}] 2024-11-23T03:45:20,128 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:20,128 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 4dbfd6cc71ec7598dc81c3407987b732, NAME => 'testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732.', STARTKEY => '', ENDKEY => ''} 2024-11-23T03:45:20,128 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732333517387 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,129 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T03:45:20,129 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,129 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,130 INFO [StoreOpener-4dbfd6cc71ec7598dc81c3407987b732-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,130 INFO [StoreOpener-4dbfd6cc71ec7598dc81c3407987b732-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dbfd6cc71ec7598dc81c3407987b732 columnFamilyName family 2024-11-23T03:45:20,130 DEBUG [StoreOpener-4dbfd6cc71ec7598dc81c3407987b732-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T03:45:20,132 DEBUG [StoreFileOpener-4dbfd6cc71ec7598dc81c3407987b732-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/family/27d97eee5ba74798b57488971c40b547, delay=913 ms 2024-11-23T03:45:20,133 DEBUG [StoreOpener-4dbfd6cc71ec7598dc81c3407987b732-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/family/27d97eee5ba74798b57488971c40b547 2024-11-23T03:45:20,133 INFO [StoreOpener-4dbfd6cc71ec7598dc81c3407987b732-1 {}] regionserver.HStore(327): Store=4dbfd6cc71ec7598dc81c3407987b732/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T03:45:20,133 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,134 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,134 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,134 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,134 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,135 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,136 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 4dbfd6cc71ec7598dc81c3407987b732; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63981326, jitterRate=-0.04660394787788391}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T03:45:20,136 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,137 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 4dbfd6cc71ec7598dc81c3407987b732: Running coprocessor pre-open hook at 1732333520129Writing region info on filesystem at 1732333520129Initializing all the Stores at 1732333520129Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732333520129Cleaning up temporary data from old regions at 1732333520134 (+5 ms)Running coprocessor post-open hooks at 1732333520136 (+2 ms)Region opened successfully at 1732333520137 (+1 ms) 2024-11-23T03:45:20,137 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732., pid=11, masterSystemTime=1732333520125 2024-11-23T03:45:20,140 DEBUG [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:20,140 INFO [RS_OPEN_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:20,140 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4dbfd6cc71ec7598dc81c3407987b732, regionState=OPEN, openSeqNum=10, regionLocation=237a1da887a0,44421,1732333516396 2024-11-23T03:45:20,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4dbfd6cc71ec7598dc81c3407987b732, server=237a1da887a0,44421,1732333516396 because future has completed 2024-11-23T03:45:20,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-23T03:45:20,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 4dbfd6cc71ec7598dc81c3407987b732, server=237a1da887a0,44421,1732333516396 in 171 msec 2024-11-23T03:45:20,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-23T03:45:20,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732333517387, region=4dbfd6cc71ec7598dc81c3407987b732, ASSIGN in 442 msec 2024-11-23T03:45:20,148 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 237a1da887a0,35247,1732333516428 after splitting done 2024-11-23T03:45:20,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 237a1da887a0,35247,1732333516428, splitWal=true, meta=true in 843 msec 2024-11-23T03:45:20,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T03:45:20,228 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T03:45:20,228 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T03:45:20,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:20,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:20,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T03:45:20,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T03:45:20,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1809726366, stopped=false 2024-11-23T03:45:20,229 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=237a1da887a0,39357,1732333516336 2024-11-23T03:45:20,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39237-0x10142eda0ac0003, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:20,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:20,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T03:45:20,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39237-0x10142eda0ac0003, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:20,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:20,231 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T03:45:20,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:20,231 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T03:45:20,231 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T03:45:20,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:20,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:20,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39237-0x10142eda0ac0003, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:20,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T03:45:20,232 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '237a1da887a0,44421,1732333516396' ***** 2024-11-23T03:45:20,232 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T03:45:20,232 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '237a1da887a0,39237,1732333519597' ***** 2024-11-23T03:45:20,232 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T03:45:20,232 INFO [RS:0;237a1da887a0:44421 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T03:45:20,232 INFO [RS:2;237a1da887a0:39237 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T03:45:20,232 INFO [RS:0;237a1da887a0:44421 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.persistence 2024-11-23T03:45:20,232 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T03:45:20,232 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T03:45:20,232 INFO [RS:2;237a1da887a0:39237 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5605d080-0d5a-ea58-b7b1-3e8a2a084c8b/bucket.persistence 2024-11-23T03:45:20,233 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T03:45:20,233 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T03:45:20,233 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T03:45:20,233 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T03:45:20,233 INFO [RS:0;237a1da887a0:44421 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T03:45:20,233 INFO [RS:2;237a1da887a0:39237 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T03:45:20,233 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T03:45:20,233 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T03:45:20,243 DEBUG [RS:0;237a1da887a0:44421 {}] bucket.BucketCache(1487): Thread RS:0;237a1da887a0:44421 started persisting bucket cache to file 2024-11-23T03:45:20,243 DEBUG [RS:2;237a1da887a0:39237 {}] bucket.BucketCache(1487): Thread RS:2;237a1da887a0:39237 started persisting bucket cache to file 2024-11-23T03:45:20,244 DEBUG [RS:0;237a1da887a0:44421 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T03:45:20,244 DEBUG [RS:2;237a1da887a0:39237 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T03:45:20,244 DEBUG [RS:0;237a1da887a0:44421 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T03:45:20,244 DEBUG [RS:2;237a1da887a0:39237 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T03:45:20,247 DEBUG [RS:0;237a1da887a0:44421 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T03:45:20,247 DEBUG [RS:0;237a1da887a0:44421 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732333520243 2024-11-23T03:45:20,247 DEBUG [RS:2;237a1da887a0:39237 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-23T03:45:20,247 DEBUG [RS:2;237a1da887a0:39237 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732333520243 2024-11-23T03:45:20,247 DEBUG [RS:0;237a1da887a0:44421 {}] bucket.BucketCache(1509): Thread RS:0;237a1da887a0:44421 finished persisting bucket cache to file, renaming 2024-11-23T03:45:20,247 DEBUG [RS:2;237a1da887a0:39237 {}] bucket.BucketCache(1509): Thread RS:2;237a1da887a0:39237 finished persisting bucket cache to file, renaming 2024-11-23T03:45:20,247 WARN [RS:2;237a1da887a0:39237 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-23T03:45:20,248 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T03:45:20,248 INFO [RS:2;237a1da887a0:39237 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T03:45:20,248 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T03:45:20,248 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T03:45:20,248 INFO [RS:2;237a1da887a0:39237 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T03:45:20,248 INFO [RS:0;237a1da887a0:44421 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T03:45:20,248 INFO [RS:2;237a1da887a0:39237 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T03:45:20,248 INFO [RS:0;237a1da887a0:44421 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T03:45:20,248 INFO [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer(959): stopping server 237a1da887a0,39237,1732333519597 2024-11-23T03:45:20,248 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(3091): Received CLOSE for 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,248 INFO [RS:2;237a1da887a0:39237 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T03:45:20,248 INFO [RS:2;237a1da887a0:39237 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;237a1da887a0:39237. 2024-11-23T03:45:20,248 DEBUG [RS:2;237a1da887a0:39237 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T03:45:20,248 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(959): stopping server 237a1da887a0,44421,1732333516396 2024-11-23T03:45:20,248 DEBUG [RS:2;237a1da887a0:39237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:20,248 INFO [RS:0;237a1da887a0:44421 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T03:45:20,248 INFO [RS:0;237a1da887a0:44421 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;237a1da887a0:44421. 2024-11-23T03:45:20,248 INFO [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer(976): stopping server 237a1da887a0,39237,1732333519597; all regions closed. 2024-11-23T03:45:20,248 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T03:45:20,248 DEBUG [RS:0;237a1da887a0:44421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T03:45:20,249 DEBUG [RS:0;237a1da887a0:44421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:20,249 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4dbfd6cc71ec7598dc81c3407987b732, disabling compactions & flushes 2024-11-23T03:45:20,249 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:20,249 INFO [RS:0;237a1da887a0:44421 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T03:45:20,249 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:20,249 INFO [RS:0;237a1da887a0:44421 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T03:45:20,249 INFO [RS:0;237a1da887a0:44421 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T03:45:20,249 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. after waiting 0 ms 2024-11-23T03:45:20,249 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T03:45:20,249 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:20,249 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T03:45:20,249 DEBUG [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(1325): Online Regions={4dbfd6cc71ec7598dc81c3407987b732=testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732., 1588230740=hbase:meta,,1.1588230740} 2024-11-23T03:45:20,249 DEBUG [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4dbfd6cc71ec7598dc81c3407987b732 2024-11-23T03:45:20,249 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T03:45:20,249 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T03:45:20,249 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T03:45:20,249 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T03:45:20,249 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T03:45:20,250 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T03:45:20,250 DEBUG [StoreFileCloser-4dbfd6cc71ec7598dc81c3407987b732-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/family/27d97eee5ba74798b57488971c40b547 2024-11-23T03:45:20,250 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-23T03:45:20,250 DEBUG [RS:2;237a1da887a0:39237 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs 2024-11-23T03:45:20,251 INFO [RS:2;237a1da887a0:39237 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 237a1da887a0%2C39237%2C1732333519597:(num 1732333519832) 2024-11-23T03:45:20,251 DEBUG [RS:2;237a1da887a0:39237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:20,251 INFO [RS:2;237a1da887a0:39237 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T03:45:20,251 INFO [RS:2;237a1da887a0:39237 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T03:45:20,251 INFO [RS:2;237a1da887a0:39237 {}] hbase.ChoreService(370): Chore service for: regionserver/237a1da887a0:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T03:45:20,251 INFO [RS:2;237a1da887a0:39237 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T03:45:20,251 INFO [RS:2;237a1da887a0:39237 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T03:45:20,251 INFO [regionserver/237a1da887a0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T03:45:20,251 INFO [RS:2;237a1da887a0:39237 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T03:45:20,251 INFO [RS:2;237a1da887a0:39237 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T03:45:20,252 INFO [RS:2;237a1da887a0:39237 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39237 2024-11-23T03:45:20,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T03:45:20,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39237-0x10142eda0ac0003, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/237a1da887a0,39237,1732333519597 2024-11-23T03:45:20,254 INFO [RS:2;237a1da887a0:39237 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T03:45:20,255 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [237a1da887a0,39237,1732333519597] 2024-11-23T03:45:20,255 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/default/testBlockEvictionOnGracefulStop1732333517387/4dbfd6cc71ec7598dc81c3407987b732/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-23T03:45:20,255 INFO [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:20,255 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4dbfd6cc71ec7598dc81c3407987b732: Waiting for close lock at 1732333520248Running coprocessor pre-close hooks at 1732333520248Disabling compacts and flushes for region at 1732333520248Disabling writes for close at 1732333520249 (+1 ms)Writing region close event to WAL at 1732333520250 (+1 ms)Running coprocessor post-close hooks at 1732333520255 (+5 ms)Closed at 1732333520255 2024-11-23T03:45:20,255 DEBUG [RS_CLOSE_REGION-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732. 2024-11-23T03:45:20,257 INFO [regionserver/237a1da887a0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T03:45:20,257 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/237a1da887a0,39237,1732333519597 already deleted, retry=false 2024-11-23T03:45:20,258 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 237a1da887a0,39237,1732333519597 expired; onlineServers=1 2024-11-23T03:45:20,261 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4c1e68252edc493b90fa0545c24b13d8 2024-11-23T03:45:20,276 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/info/4c1e68252edc493b90fa0545c24b13d8 is 209, key is testBlockEvictionOnGracefulStop1732333517387,,1732333517392.4dbfd6cc71ec7598dc81c3407987b732./info:regioninfo/1732333520140/Put/seqid=0 2024-11-23T03:45:20,276 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/info/4c1e68252edc493b90fa0545c24b13d8 2024-11-23T03:45:20,278 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/.tmp/info/4c1e68252edc493b90fa0545c24b13d8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/4c1e68252edc493b90fa0545c24b13d8 2024-11-23T03:45:20,279 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/4c1e68252edc493b90fa0545c24b13d8, delay=999 ms 2024-11-23T03:45:20,280 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/4c1e68252edc493b90fa0545c24b13d8, entries=8, sequenceid=19, filesize=6.6 K 2024-11-23T03:45:20,281 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false 2024-11-23T03:45:20,282 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/3a7b5eee8d7745eb9558f4ece26764e1 2024-11-23T03:45:20,282 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/info/4c1e68252edc493b90fa0545c24b13d8 2024-11-23T03:45:20,283 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/ns/a8575720a67848ee81fe3ccae9e4f04d 2024-11-23T03:45:20,283 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/table/56dae1eac8184e578e886de68acba834 2024-11-23T03:45:20,285 INFO [regionserver/237a1da887a0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T03:45:20,287 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-23T03:45:20,287 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T03:45:20,287 INFO [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T03:45:20,288 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732333520249Running coprocessor pre-close hooks at 1732333520249Disabling compacts and flushes for region at 1732333520249Disabling writes for close at 1732333520250 (+1 ms)Obtaining lock to block concurrent updates at 1732333520250Preparing flush snapshotting stores in 1588230740 at 1732333520250Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732333520251 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732333520251Flushing 1588230740/info: creating writer at 1732333520251Flushing 1588230740/info: appending metadata at 1732333520275 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732333520275Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4caed846: reopening flushed file at 1732333520278 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false at 1732333520281 (+3 ms)Writing region close event to WAL at 1732333520283 (+2 ms)Running coprocessor post-close hooks at 1732333520287 (+4 ms)Closed at 1732333520287 2024-11-23T03:45:20,288 DEBUG [RS_CLOSE_META-regionserver/237a1da887a0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T03:45:20,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39237-0x10142eda0ac0003, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:20,357 INFO [RS:2;237a1da887a0:39237 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T03:45:20,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39237-0x10142eda0ac0003, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:20,357 INFO [RS:2;237a1da887a0:39237 {}] regionserver.HRegionServer(1031): Exiting; stopping=237a1da887a0,39237,1732333519597; zookeeper connection closed. 2024-11-23T03:45:20,357 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@33aa8263 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@33aa8263 2024-11-23T03:45:20,449 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(976): stopping server 237a1da887a0,44421,1732333516396; all regions closed. 2024-11-23T03:45:20,451 DEBUG [RS:0;237a1da887a0:44421 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs 2024-11-23T03:45:20,451 INFO [RS:0;237a1da887a0:44421 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 237a1da887a0%2C44421%2C1732333516396.meta:.meta(num 1732333519638) 2024-11-23T03:45:20,453 DEBUG [RS:0;237a1da887a0:44421 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/oldWALs 2024-11-23T03:45:20,453 INFO [RS:0;237a1da887a0:44421 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 237a1da887a0%2C44421%2C1732333516396:(num 1732333517000) 2024-11-23T03:45:20,453 DEBUG [RS:0;237a1da887a0:44421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T03:45:20,453 INFO [RS:0;237a1da887a0:44421 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T03:45:20,453 INFO [RS:0;237a1da887a0:44421 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T03:45:20,453 INFO [RS:0;237a1da887a0:44421 {}] hbase.ChoreService(370): Chore service for: regionserver/237a1da887a0:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T03:45:20,453 INFO [RS:0;237a1da887a0:44421 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T03:45:20,453 INFO [regionserver/237a1da887a0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T03:45:20,454 INFO [RS:0;237a1da887a0:44421 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44421 2024-11-23T03:45:20,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/237a1da887a0,44421,1732333516396 2024-11-23T03:45:20,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T03:45:20,456 INFO [RS:0;237a1da887a0:44421 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T03:45:20,458 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [237a1da887a0,44421,1732333516396] 2024-11-23T03:45:20,460 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/237a1da887a0,44421,1732333516396 already deleted, retry=false 2024-11-23T03:45:20,460 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 237a1da887a0,44421,1732333516396 expired; onlineServers=0 2024-11-23T03:45:20,460 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '237a1da887a0,39357,1732333516336' ***** 2024-11-23T03:45:20,460 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T03:45:20,461 INFO [M:0;237a1da887a0:39357 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T03:45:20,461 INFO [M:0;237a1da887a0:39357 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T03:45:20,461 DEBUG [M:0;237a1da887a0:39357 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T03:45:20,461 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T03:45:20,461 DEBUG [M:0;237a1da887a0:39357 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T03:45:20,461 DEBUG [master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.large.0-1732333516785 {}] cleaner.HFileCleaner(306): Exit Thread[master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.large.0-1732333516785,5,FailOnTimeoutGroup] 2024-11-23T03:45:20,461 DEBUG [master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.small.0-1732333516785 {}] cleaner.HFileCleaner(306): Exit Thread[master/237a1da887a0:0:becomeActiveMaster-HFileCleaner.small.0-1732333516785,5,FailOnTimeoutGroup] 2024-11-23T03:45:20,461 INFO [M:0;237a1da887a0:39357 {}] hbase.ChoreService(370): Chore service for: master/237a1da887a0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T03:45:20,461 INFO [M:0;237a1da887a0:39357 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T03:45:20,461 DEBUG [M:0;237a1da887a0:39357 {}] master.HMaster(1795): Stopping service threads 2024-11-23T03:45:20,461 INFO [M:0;237a1da887a0:39357 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T03:45:20,462 INFO [M:0;237a1da887a0:39357 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T03:45:20,462 INFO [M:0;237a1da887a0:39357 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T03:45:20,462 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T03:45:20,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T03:45:20,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T03:45:20,463 DEBUG [M:0;237a1da887a0:39357 {}] zookeeper.ZKUtil(347): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T03:45:20,463 WARN [M:0;237a1da887a0:39357 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T03:45:20,463 INFO [M:0;237a1da887a0:39357 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/.lastflushedseqids 2024-11-23T03:45:20,470 INFO [M:0;237a1da887a0:39357 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T03:45:20,470 INFO [M:0;237a1da887a0:39357 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T03:45:20,470 DEBUG [M:0;237a1da887a0:39357 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T03:45:20,470 INFO [M:0;237a1da887a0:39357 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:20,470 DEBUG [M:0;237a1da887a0:39357 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:20,470 DEBUG [M:0;237a1da887a0:39357 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T03:45:20,470 DEBUG [M:0;237a1da887a0:39357 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:20,470 INFO [M:0;237a1da887a0:39357 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-23T03:45:20,480 DEBUG [M:0;237a1da887a0:39357 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6584abe7fdc44d12afe8eafac5b42b9c 2024-11-23T03:45:20,494 DEBUG [M:0;237a1da887a0:39357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6584abe7fdc44d12afe8eafac5b42b9c is 82, key is hbase:meta,,1/info:regioninfo/1732333519691/Put/seqid=0 2024-11-23T03:45:20,495 INFO [M:0;237a1da887a0:39357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6584abe7fdc44d12afe8eafac5b42b9c 2024-11-23T03:45:20,502 DEBUG [M:0;237a1da887a0:39357 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d14b22e12f984930b48383f487fdc916 2024-11-23T03:45:20,517 DEBUG [M:0;237a1da887a0:39357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d14b22e12f984930b48383f487fdc916 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732333517808/Put/seqid=0 2024-11-23T03:45:20,517 INFO [M:0;237a1da887a0:39357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d14b22e12f984930b48383f487fdc916 2024-11-23T03:45:20,525 DEBUG [M:0;237a1da887a0:39357 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f4624e61fa62489894f8b899864aa593 2024-11-23T03:45:20,539 DEBUG [M:0;237a1da887a0:39357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f4624e61fa62489894f8b899864aa593 is 69, key is 237a1da887a0,39237,1732333519597/rs:state/1732333519655/Put/seqid=0 2024-11-23T03:45:20,540 INFO [M:0;237a1da887a0:39357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f4624e61fa62489894f8b899864aa593 2024-11-23T03:45:20,542 INFO [M:0;237a1da887a0:39357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f4624e61fa62489894f8b899864aa593 2024-11-23T03:45:20,542 DEBUG [M:0;237a1da887a0:39357 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6584abe7fdc44d12afe8eafac5b42b9c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6584abe7fdc44d12afe8eafac5b42b9c 2024-11-23T03:45:20,543 INFO [M:0;237a1da887a0:39357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6584abe7fdc44d12afe8eafac5b42b9c, entries=14, sequenceid=102, filesize=5.9 K 2024-11-23T03:45:20,544 DEBUG [M:0;237a1da887a0:39357 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d14b22e12f984930b48383f487fdc916 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d14b22e12f984930b48383f487fdc916 2024-11-23T03:45:20,545 INFO [M:0;237a1da887a0:39357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d14b22e12f984930b48383f487fdc916, entries=11, sequenceid=102, filesize=6.5 K 2024-11-23T03:45:20,546 DEBUG [M:0;237a1da887a0:39357 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f4624e61fa62489894f8b899864aa593 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f4624e61fa62489894f8b899864aa593 2024-11-23T03:45:20,547 INFO [M:0;237a1da887a0:39357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f4624e61fa62489894f8b899864aa593 2024-11-23T03:45:20,547 INFO [M:0;237a1da887a0:39357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af59af5d-82bc-5f4c-134f-3cb40b7c1054/d9bd8913-573c-fe3e-e217-c15e63b395dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f4624e61fa62489894f8b899864aa593, entries=3, sequenceid=102, filesize=5.3 K 2024-11-23T03:45:20,548 INFO [M:0;237a1da887a0:39357 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=102, compaction requested=false 2024-11-23T03:45:20,550 INFO [M:0;237a1da887a0:39357 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T03:45:20,550 DEBUG [M:0;237a1da887a0:39357 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732333520470Disabling compacts and flushes for region at 1732333520470Disabling writes for close at 1732333520470Obtaining lock to block concurrent updates at 1732333520470Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732333520470Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732333520471 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732333520471Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732333520471Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732333520494 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732333520494Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732333520497 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732333520516 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732333520516Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732333520519 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732333520538 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732333520538Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dff9a29: reopening flushed file at 1732333520542 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78bcb107: reopening flushed file at 1732333520544 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c00985f: reopening flushed file at 1732333520545 (+1 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=102, compaction requested=false at 1732333520548 (+3 ms)Writing region close event to WAL at 1732333520550 (+2 ms)Closed at 1732333520550 2024-11-23T03:45:20,552 INFO [M:0;237a1da887a0:39357 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T03:45:20,552 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T03:45:20,552 INFO [M:0;237a1da887a0:39357 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39357 2024-11-23T03:45:20,552 INFO [M:0;237a1da887a0:39357 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T03:45:20,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:20,558 INFO [RS:0;237a1da887a0:44421 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T03:45:20,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44421-0x10142eda0ac0001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:20,558 INFO [RS:0;237a1da887a0:44421 {}] regionserver.HRegionServer(1031): Exiting; stopping=237a1da887a0,44421,1732333516396; zookeeper connection closed. 2024-11-23T03:45:20,558 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7e0096db {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7e0096db 2024-11-23T03:45:20,558 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-23T03:45:20,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:20,654 INFO [M:0;237a1da887a0:39357 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T03:45:20,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39357-0x10142eda0ac0000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T03:45:20,657 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T03:45:20,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T03:45:20,658 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T03:45:20,665 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 334), ProcessCount=11 (was 11), AvailableMemoryMB=9703 (was 10002)