2024-11-14 14:24:32,480 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-14 14:24:32,498 main DEBUG Took 0.016853 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 14:24:32,499 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 14:24:32,499 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 14:24:32,500 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 14:24:32,501 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,508 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 14:24:32,520 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,522 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,522 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,523 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,524 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,524 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,525 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,526 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,526 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,526 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,527 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,527 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,528 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,528 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,528 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,529 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,529 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,529 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,530 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,530 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:24:32,530 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,531 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 14:24:32,532 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:24:32,534 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 14:24:32,535 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 14:24:32,536 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 14:24:32,537 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 14:24:32,537 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 14:24:32,545 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 14:24:32,547 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 14:24:32,549 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 14:24:32,549 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 14:24:32,550 main DEBUG createAppenders(={Console}) 2024-11-14 14:24:32,551 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-14 14:24:32,551 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-14 14:24:32,551 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-14 14:24:32,552 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 14:24:32,552 main DEBUG OutputStream closed 2024-11-14 14:24:32,552 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 14:24:32,552 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 14:24:32,553 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-14 14:24:32,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 14:24:32,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 14:24:32,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 14:24:32,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 14:24:32,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 14:24:32,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 14:24:32,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 14:24:32,624 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 14:24:32,624 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 14:24:32,624 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 14:24:32,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 14:24:32,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 14:24:32,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 14:24:32,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 14:24:32,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 14:24:32,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 14:24:32,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 14:24:32,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 14:24:32,629 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 14:24:32,629 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-14 14:24:32,629 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 14:24:32,630 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-14T14:24:32,641 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-14 14:24:32,643 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 14:24:32,643 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T14:24:32,901 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c 2024-11-14T14:24:32,926 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=4773 2024-11-14T14:24:33,008 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T14:24:33,093 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:24:33,107 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/cluster_30c99684-aaa2-cedb-9cbd-af4213d882e5, deleteOnExit=true 2024-11-14T14:24:33,192 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/cluster_30c99684-aaa2-cedb-9cbd-af4213d882e5/zookeeper_0, clientPort=50137, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/cluster_30c99684-aaa2-cedb-9cbd-af4213d882e5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/cluster_30c99684-aaa2-cedb-9cbd-af4213d882e5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T14:24:33,205 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50137 2024-11-14T14:24:33,205 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:24:33,301 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb with version=8 2024-11-14T14:24:33,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/hbase-staging 2024-11-14T14:24:33,390 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T14:24:33,640 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6e2cc31459a7:0 server-side Connection retries=45 2024-11-14T14:24:33,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:24:33,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:24:33,657 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:24:33,657 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:24:33,657 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:24:33,810 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T14:24:33,889 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T14:24:33,934 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T14:24:33,945 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T14:24:33,949 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:24:33,986 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 35818 (auto-detected) 2024-11-14T14:24:33,987 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T14:24:34,010 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42801 2024-11-14T14:24:34,031 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42801 connecting to ZooKeeper ensemble=127.0.0.1:50137 2024-11-14T14:24:34,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428010x0, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:24:34,118 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42801-0x10139bbbd3e0000 connected 2024-11-14T14:24:34,338 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:24:34,339 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:24:34,348 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:24:34,353 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb, hbase.cluster.distributed=false 2024-11-14T14:24:34,378 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:24:34,384 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42801 2024-11-14T14:24:34,384 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42801 2024-11-14T14:24:34,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42801 2024-11-14T14:24:34,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42801 2024-11-14T14:24:34,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42801 2024-11-14T14:24:34,457 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6e2cc31459a7:0 server-side Connection retries=45 2024-11-14T14:24:34,459 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:24:34,460 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:24:34,460 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:24:34,460 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:24:34,460 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:24:34,463 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:24:34,467 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:24:34,468 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39749 2024-11-14T14:24:34,471 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39749 connecting to ZooKeeper ensemble=127.0.0.1:50137 2024-11-14T14:24:34,471 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:24:34,472 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:24:34,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:397490x0, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:24:34,482 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:397490x0, quorum=127.0.0.1:50137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:24:34,482 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39749-0x10139bbbd3e0001 connected 2024-11-14T14:24:34,488 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:24:34,499 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bucket.cache 2024-11-14T14:24:34,505 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T14:24:34,506 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T14:24:34,507 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-14T14:24:34,509 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T14:24:34,509 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-14T14:24:34,516 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T14:24:34,520 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T14:24:34,521 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bucket.persistence is complete. 2024-11-14T14:24:34,522 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:24:34,525 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:24:34,530 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:24:34,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39749 2024-11-14T14:24:34,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39749 2024-11-14T14:24:34,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39749 2024-11-14T14:24:34,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39749 2024-11-14T14:24:34,536 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39749 2024-11-14T14:24:34,552 DEBUG [M:0;6e2cc31459a7:42801 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6e2cc31459a7:42801 2024-11-14T14:24:34,553 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6e2cc31459a7,42801,1731594273477 2024-11-14T14:24:34,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:24:34,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:24:34,575 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6e2cc31459a7,42801,1731594273477 2024-11-14T14:24:34,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:24:34,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:34,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:34,605 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T14:24:34,607 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6e2cc31459a7,42801,1731594273477 from backup master directory 2024-11-14T14:24:34,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:24:34,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6e2cc31459a7,42801,1731594273477 2024-11-14T14:24:34,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:24:34,619 WARN [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:24:34,619 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6e2cc31459a7,42801,1731594273477 2024-11-14T14:24:34,621 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T14:24:34,623 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T14:24:34,631 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/hbase.id] with ID: f2d3e065-75e2-4d35-93cb-cb543d9b4422 2024-11-14T14:24:34,631 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/.tmp/hbase.id 2024-11-14T14:24:34,636 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/hbase.id] 2024-11-14T14:24:34,672 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:24:34,677 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T14:24:34,698 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-14T14:24:34,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:34,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:34,737 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:24:34,741 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T14:24:34,779 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:24:34,784 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:24:34,843 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store 2024-11-14T14:24:34,870 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T14:24:34,874 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:24:34,876 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:24:34,877 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:24:34,877 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:24:34,880 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:24:34,880 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:24:34,881 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:24:34,882 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731594274876Disabling compacts and flushes for region at 1731594274876Disabling writes for close at 1731594274880 (+4 ms)Writing region close event to WAL at 1731594274881 (+1 ms)Closed at 1731594274881 2024-11-14T14:24:34,883 WARN [master/6e2cc31459a7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/.initializing 2024-11-14T14:24:34,883 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/WALs/6e2cc31459a7,42801,1731594273477 2024-11-14T14:24:34,897 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:24:34,914 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6e2cc31459a7%2C42801%2C1731594273477, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/WALs/6e2cc31459a7,42801,1731594273477, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/oldWALs, maxLogs=10 2024-11-14T14:24:34,963 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/WALs/6e2cc31459a7,42801,1731594273477/6e2cc31459a7%2C42801%2C1731594273477.1731594274920 2024-11-14T14:24:34,964 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T14:24:34,965 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:24:34,966 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:24:34,970 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:34,972 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:35,017 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:35,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T14:24:35,057 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:35,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:24:35,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:35,070 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T14:24:35,070 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:35,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:24:35,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:35,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T14:24:35,077 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:35,078 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:24:35,078 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:35,085 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T14:24:35,085 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:35,086 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:24:35,086 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:35,089 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:35,090 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:35,093 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:35,094 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:35,096 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:24:35,098 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:24:35,104 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:24:35,105 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60672474, jitterRate=-0.09590968489646912}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:24:35,112 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731594274989Initializing all the Stores at 1731594274990 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731594274990Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731594274992 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731594274993 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731594274993Cleaning up temporary data from old regions at 1731594275094 (+101 ms)Region opened successfully at 1731594275112 (+18 ms) 2024-11-14T14:24:35,114 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T14:24:35,153 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7c3f45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6e2cc31459a7/172.17.0.2:0 2024-11-14T14:24:35,195 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T14:24:35,212 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T14:24:35,212 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T14:24:35,216 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:24:35,217 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T14:24:35,225 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-14T14:24:35,226 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T14:24:35,258 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T14:24:35,270 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T14:24:35,355 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T14:24:35,358 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T14:24:35,361 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T14:24:35,372 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T14:24:35,375 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T14:24:35,380 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T14:24:35,393 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T14:24:35,396 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T14:24:35,407 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T14:24:35,432 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T14:24:35,439 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T14:24:35,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:24:35,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:35,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:24:35,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:35,463 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6e2cc31459a7,42801,1731594273477, sessionid=0x10139bbbd3e0000, setting cluster-up flag (Was=false) 2024-11-14T14:24:35,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:35,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:35,523 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T14:24:35,526 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6e2cc31459a7,42801,1731594273477 2024-11-14T14:24:35,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:35,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:35,587 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T14:24:35,589 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6e2cc31459a7,42801,1731594273477 2024-11-14T14:24:35,597 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T14:24:35,641 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(746): ClusterId : f2d3e065-75e2-4d35-93cb-cb543d9b4422 2024-11-14T14:24:35,644 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:24:35,660 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:24:35,661 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:24:35,673 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:24:35,674 DEBUG [RS:0;6e2cc31459a7:39749 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665d7266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6e2cc31459a7/172.17.0.2:0 2024-11-14T14:24:35,693 DEBUG [RS:0;6e2cc31459a7:39749 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6e2cc31459a7:39749 2024-11-14T14:24:35,698 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:24:35,698 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:24:35,698 DEBUG [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:24:35,701 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(2659): reportForDuty to master=6e2cc31459a7,42801,1731594273477 with port=39749, startcode=1731594274426 2024-11-14T14:24:35,704 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:24:35,717 DEBUG [RS:0;6e2cc31459a7:39749 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:24:35,719 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T14:24:35,728 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T14:24:35,736 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6e2cc31459a7,42801,1731594273477 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T14:24:35,745 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6e2cc31459a7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:24:35,746 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6e2cc31459a7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:24:35,746 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6e2cc31459a7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:24:35,746 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6e2cc31459a7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:24:35,747 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6e2cc31459a7:0, corePoolSize=10, maxPoolSize=10 2024-11-14T14:24:35,747 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,747 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6e2cc31459a7:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:24:35,747 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,761 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:24:35,762 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T14:24:35,767 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:35,767 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T14:24:35,772 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731594305772 2024-11-14T14:24:35,777 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T14:24:35,781 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T14:24:35,785 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T14:24:35,786 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T14:24:35,786 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T14:24:35,787 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T14:24:35,800 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:35,813 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T14:24:35,815 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T14:24:35,815 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T14:24:35,818 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T14:24:35,819 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T14:24:35,821 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6e2cc31459a7:0:becomeActiveMaster-HFileCleaner.large.0-1731594275821,5,FailOnTimeoutGroup] 2024-11-14T14:24:35,822 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39633, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:24:35,822 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6e2cc31459a7:0:becomeActiveMaster-HFileCleaner.small.0-1731594275822,5,FailOnTimeoutGroup] 2024-11-14T14:24:35,822 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:35,822 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T14:24:35,824 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:35,824 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:35,828 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42801 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:35,830 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42801 {}] master.ServerManager(517): Registering regionserver=6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:35,847 DEBUG [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb 2024-11-14T14:24:35,848 DEBUG [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T14:24:35,848 DEBUG [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:24:35,850 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T14:24:35,851 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb 2024-11-14T14:24:35,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:24:35,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T14:24:35,881 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:24:35,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:35,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:24:35,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T14:24:35,887 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:24:35,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:35,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:24:35,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T14:24:35,892 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:24:35,892 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:35,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:24:35,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T14:24:35,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:24:35,895 DEBUG [RS:0;6e2cc31459a7:39749 {}] zookeeper.ZKUtil(111): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:35,895 WARN [RS:0;6e2cc31459a7:39749 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:24:35,895 INFO [RS:0;6e2cc31459a7:39749 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:24:35,895 DEBUG [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/WALs/6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:35,897 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6e2cc31459a7,39749,1731594274426] 2024-11-14T14:24:35,908 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:24:35,909 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:35,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:24:35,909 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:24:35,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740 2024-11-14T14:24:35,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740 2024-11-14T14:24:35,913 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:24:35,913 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:24:35,913 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:24:35,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:24:35,929 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:24:35,929 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59858730, jitterRate=-0.10803541541099548}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:24:35,934 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:24:35,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731594275862Initializing all the Stores at 1731594275863 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731594275863Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731594275863Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731594275863Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731594275863Cleaning up temporary data from old regions at 1731594275913 (+50 ms)Region opened successfully at 1731594275934 (+21 ms) 2024-11-14T14:24:35,935 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:24:35,935 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:24:35,935 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:24:35,935 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:24:35,936 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:24:35,938 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:24:35,938 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731594275935Disabling compacts and flushes for region at 1731594275935Disabling writes for close at 1731594275936 (+1 ms)Writing region close event to WAL at 1731594275937 (+1 ms)Closed at 1731594275937 2024-11-14T14:24:35,941 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:24:35,942 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T14:24:35,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:24:35,962 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:24:35,968 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:24:35,974 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T14:24:35,980 INFO [RS:0;6e2cc31459a7:39749 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:24:35,980 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:35,984 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:24:35,992 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:24:35,994 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:35,995 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,995 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,996 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,996 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,996 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,996 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6e2cc31459a7:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:24:35,997 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,997 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,997 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,997 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,997 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,998 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6e2cc31459a7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:24:35,998 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6e2cc31459a7:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:24:35,998 DEBUG [RS:0;6e2cc31459a7:39749 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6e2cc31459a7:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:24:36,004 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:36,005 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:36,005 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:36,006 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:36,006 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:36,007 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(168): Chore ScheduledChore name=6e2cc31459a7,39749,1731594274426-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:24:36,035 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:24:36,037 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(168): Chore ScheduledChore name=6e2cc31459a7,39749,1731594274426-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:36,038 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:36,038 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.Replication(171): 6e2cc31459a7,39749,1731594274426 started 2024-11-14T14:24:36,066 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:36,067 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(1482): Serving as 6e2cc31459a7,39749,1731594274426, RpcServer on 6e2cc31459a7/172.17.0.2:39749, sessionid=0x10139bbbd3e0001 2024-11-14T14:24:36,068 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:24:36,068 DEBUG [RS:0;6e2cc31459a7:39749 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:36,069 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6e2cc31459a7,39749,1731594274426' 2024-11-14T14:24:36,069 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:24:36,070 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:24:36,072 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:24:36,072 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:24:36,072 DEBUG [RS:0;6e2cc31459a7:39749 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:36,072 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6e2cc31459a7,39749,1731594274426' 2024-11-14T14:24:36,072 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:24:36,073 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:24:36,074 DEBUG [RS:0;6e2cc31459a7:39749 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:24:36,075 INFO [RS:0;6e2cc31459a7:39749 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:24:36,075 INFO [RS:0;6e2cc31459a7:39749 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:24:36,126 WARN [6e2cc31459a7:42801 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T14:24:36,180 INFO [RS:0;6e2cc31459a7:39749 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:24:36,182 INFO [RS:0;6e2cc31459a7:39749 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6e2cc31459a7%2C39749%2C1731594274426, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/WALs/6e2cc31459a7,39749,1731594274426, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/oldWALs, maxLogs=55 2024-11-14T14:24:36,205 INFO [RS:0;6e2cc31459a7:39749 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/WALs/6e2cc31459a7,39749,1731594274426/6e2cc31459a7%2C39749%2C1731594274426.1731594276184 2024-11-14T14:24:36,206 DEBUG [RS:0;6e2cc31459a7:39749 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T14:24:36,379 DEBUG [6e2cc31459a7:42801 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T14:24:36,395 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:36,402 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6e2cc31459a7,39749,1731594274426, state=OPENING 2024-11-14T14:24:36,449 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T14:24:36,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:36,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:36,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:24:36,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:24:36,464 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:24:36,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6e2cc31459a7,39749,1731594274426}] 2024-11-14T14:24:36,648 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:24:36,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52937, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:24:36,665 INFO [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T14:24:36,666 INFO [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:24:36,668 INFO [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T14:24:36,670 INFO [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6e2cc31459a7%2C39749%2C1731594274426.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/WALs/6e2cc31459a7,39749,1731594274426, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/oldWALs, maxLogs=55 2024-11-14T14:24:36,699 INFO [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/WALs/6e2cc31459a7,39749,1731594274426/6e2cc31459a7%2C39749%2C1731594274426.meta.1731594276673.meta 2024-11-14T14:24:36,699 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T14:24:36,700 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:24:36,703 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T14:24:36,707 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T14:24:36,714 INFO [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T14:24:36,720 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T14:24:36,721 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:24:36,721 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T14:24:36,721 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T14:24:36,724 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T14:24:36,725 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:24:36,726 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:36,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:24:36,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T14:24:36,728 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:24:36,728 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:36,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:24:36,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T14:24:36,730 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:24:36,730 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:36,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:24:36,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T14:24:36,732 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:24:36,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:36,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:24:36,733 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:24:36,734 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740 2024-11-14T14:24:36,736 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740 2024-11-14T14:24:36,736 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:24:36,736 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:24:36,737 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:24:36,739 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:24:36,740 INFO [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62618462, jitterRate=-0.0669122040271759}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:24:36,740 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T14:24:36,743 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731594276722Writing region info on filesystem at 1731594276723 (+1 ms)Initializing all the Stores at 1731594276723Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731594276723Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731594276724 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731594276724Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731594276724Cleaning up temporary data from old regions at 1731594276737 (+13 ms)Running coprocessor post-open hooks at 1731594276740 (+3 ms)Region opened successfully at 1731594276742 (+2 ms) 2024-11-14T14:24:36,750 INFO [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731594276637 2024-11-14T14:24:36,766 DEBUG [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T14:24:36,767 INFO [RS_OPEN_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T14:24:36,768 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:36,771 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6e2cc31459a7,39749,1731594274426, state=OPEN 2024-11-14T14:24:36,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:24:36,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:24:36,894 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:24:36,894 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:24:36,895 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:36,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T14:24:36,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6e2cc31459a7,39749,1731594274426 in 428 msec 2024-11-14T14:24:36,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T14:24:36,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 950 msec 2024-11-14T14:24:36,907 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:24:36,907 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T14:24:36,933 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:24:36,934 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6e2cc31459a7,39749,1731594274426, seqNum=-1] 2024-11-14T14:24:36,958 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:24:36,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39659, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:24:36,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3390 sec 2024-11-14T14:24:36,983 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731594276983, completionTime=-1 2024-11-14T14:24:36,986 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T14:24:36,986 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T14:24:37,012 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-14T14:24:37,012 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731594337012 2024-11-14T14:24:37,012 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731594397012 2024-11-14T14:24:37,012 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 25 msec 2024-11-14T14:24:37,015 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6e2cc31459a7,42801,1731594273477-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:37,015 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6e2cc31459a7,42801,1731594273477-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:37,016 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6e2cc31459a7,42801,1731594273477-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:37,017 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6e2cc31459a7:42801, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:37,017 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:37,018 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:37,024 DEBUG [master/6e2cc31459a7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T14:24:37,047 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.427sec 2024-11-14T14:24:37,048 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T14:24:37,049 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T14:24:37,050 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T14:24:37,050 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T14:24:37,051 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T14:24:37,051 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6e2cc31459a7,42801,1731594273477-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:24:37,051 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6e2cc31459a7,42801,1731594273477-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T14:24:37,055 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T14:24:37,056 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T14:24:37,056 INFO [master/6e2cc31459a7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6e2cc31459a7,42801,1731594273477-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:24:37,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1efa262a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:24:37,155 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T14:24:37,155 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T14:24:37,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6e2cc31459a7,42801,-1 for getting cluster id 2024-11-14T14:24:37,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T14:24:37,174 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f2d3e065-75e2-4d35-93cb-cb543d9b4422' 2024-11-14T14:24:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T14:24:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f2d3e065-75e2-4d35-93cb-cb543d9b4422" 2024-11-14T14:24:37,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd721a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:24:37,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6e2cc31459a7,42801,-1] 2024-11-14T14:24:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T14:24:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:24:37,186 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T14:24:37,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d87a917, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:24:37,189 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:24:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6e2cc31459a7,39749,1731594274426, seqNum=-1] 2024-11-14T14:24:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:24:37,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:24:37,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6e2cc31459a7,42801,1731594273477 2024-11-14T14:24:37,227 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T14:24:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6e2cc31459a7,42801,1731594273477 2024-11-14T14:24:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@594d9a31 2024-11-14T14:24:37,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T14:24:37,248 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T14:24:37,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:24:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-14T14:24:37,268 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T14:24:37,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-14T14:24:37,271 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:37,272 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T14:24:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:24:37,286 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 550c53a10099b13fa7c462d5a83e9804, NAME => 'table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb 2024-11-14T14:24:37,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:24:37,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing 550c53a10099b13fa7c462d5a83e9804, disabling compactions & flushes 2024-11-14T14:24:37,293 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:37,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:37,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. after waiting 0 ms 2024-11-14T14:24:37,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:37,293 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:37,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for 550c53a10099b13fa7c462d5a83e9804: Waiting for close lock at 1731594277293Disabling compacts and flushes for region at 1731594277293Disabling writes for close at 1731594277293Writing region close event to WAL at 1731594277293Closed at 1731594277293 2024-11-14T14:24:37,295 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T14:24:37,300 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1731594277295"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731594277295"}]},"ts":"1731594277295"} 2024-11-14T14:24:37,305 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T14:24:37,307 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T14:24:37,309 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731594277307"}]},"ts":"1731594277307"} 2024-11-14T14:24:37,314 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-14T14:24:37,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=550c53a10099b13fa7c462d5a83e9804, ASSIGN}] 2024-11-14T14:24:37,318 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=550c53a10099b13fa7c462d5a83e9804, ASSIGN 2024-11-14T14:24:37,319 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=550c53a10099b13fa7c462d5a83e9804, ASSIGN; state=OFFLINE, location=6e2cc31459a7,39749,1731594274426; forceNewPlan=false, retain=false 2024-11-14T14:24:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:24:37,471 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=550c53a10099b13fa7c462d5a83e9804, regionState=OPENING, regionLocation=6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:37,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=550c53a10099b13fa7c462d5a83e9804, ASSIGN because future has completed 2024-11-14T14:24:37,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550c53a10099b13fa7c462d5a83e9804, server=6e2cc31459a7,39749,1731594274426}] 2024-11-14T14:24:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:24:37,634 INFO [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:37,635 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 550c53a10099b13fa7c462d5a83e9804, NAME => 'table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:24:37,636 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:37,636 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:24:37,636 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:37,637 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:37,638 INFO [StoreOpener-550c53a10099b13fa7c462d5a83e9804-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:37,643 INFO [StoreOpener-550c53a10099b13fa7c462d5a83e9804-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 550c53a10099b13fa7c462d5a83e9804 columnFamilyName family 2024-11-14T14:24:37,644 DEBUG [StoreOpener-550c53a10099b13fa7c462d5a83e9804-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:24:37,644 INFO [StoreOpener-550c53a10099b13fa7c462d5a83e9804-1 {}] regionserver.HStore(327): Store=550c53a10099b13fa7c462d5a83e9804/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:24:37,645 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:37,646 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:37,646 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:37,646 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:37,647 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:37,648 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:37,655 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:24:37,655 INFO [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 550c53a10099b13fa7c462d5a83e9804; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68611387, jitterRate=0.022389337420463562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:24:37,656 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:37,657 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 550c53a10099b13fa7c462d5a83e9804: Running coprocessor pre-open hook at 1731594277637Writing region info on filesystem at 1731594277637Initializing all the Stores at 1731594277638 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731594277638Cleaning up temporary data from old regions at 1731594277647 (+9 ms)Running coprocessor post-open hooks at 1731594277656 (+9 ms)Region opened successfully at 1731594277657 (+1 ms) 2024-11-14T14:24:37,658 INFO [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804., pid=6, masterSystemTime=1731594277630 2024-11-14T14:24:37,662 DEBUG [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:37,662 INFO [RS_OPEN_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:37,662 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=550c53a10099b13fa7c462d5a83e9804, regionState=OPEN, openSeqNum=2, regionLocation=6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:37,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550c53a10099b13fa7c462d5a83e9804, server=6e2cc31459a7,39749,1731594274426 because future has completed 2024-11-14T14:24:37,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T14:24:37,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 550c53a10099b13fa7c462d5a83e9804, server=6e2cc31459a7,39749,1731594274426 in 190 msec 2024-11-14T14:24:37,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T14:24:37,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=550c53a10099b13fa7c462d5a83e9804, ASSIGN in 353 msec 2024-11-14T14:24:37,673 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T14:24:37,673 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731594277673"}]},"ts":"1731594277673"} 2024-11-14T14:24:37,675 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-14T14:24:37,677 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T14:24:37,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 415 msec 2024-11-14T14:24:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:24:37,905 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-14T14:24:37,905 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-14T14:24:37,908 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:24:37,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-14T14:24:37,917 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:24:37,918 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-14T14:24:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804., hostname=6e2cc31459a7,39749,1731594274426, seqNum=2] 2024-11-14T14:24:37,945 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 550c53a10099b13fa7c462d5a83e9804 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-14T14:24:37,989 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8d40b3587525458c9ba7ef76f4205c92 2024-11-14T14:24:38,043 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/.tmp/family/8d40b3587525458c9ba7ef76f4205c92 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-14T14:24:38,049 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/.tmp/family/8d40b3587525458c9ba7ef76f4205c92 2024-11-14T14:24:38,085 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/.tmp/family/8d40b3587525458c9ba7ef76f4205c92 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/family/8d40b3587525458c9ba7ef76f4205c92 2024-11-14T14:24:38,087 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/family/8d40b3587525458c9ba7ef76f4205c92, delay=990 ms 2024-11-14T14:24:38,090 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/family/8d40b3587525458c9ba7ef76f4205c92, entries=2, sequenceid=6, filesize=4.9 K 2024-11-14T14:24:38,099 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 550c53a10099b13fa7c462d5a83e9804 in 153ms, sequenceid=6, compaction requested=false 2024-11-14T14:24:38,100 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-14T14:24:38,103 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 550c53a10099b13fa7c462d5a83e9804: 2024-11-14T14:24:39,079 DEBUG [hfile-prefetch-1731594278087 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8d40b3587525458c9ba7ef76f4205c92_0. 2024-11-14T14:24:39,083 DEBUG [hfile-prefetch-1731594278087 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8d40b3587525458c9ba7ef76f4205c92_117. 2024-11-14T14:24:39,084 DEBUG [hfile-prefetch-1731594278087 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/family/8d40b3587525458c9ba7ef76f4205c92, with total blocks 2, and data blocks 1 2024-11-14T14:24:39,084 DEBUG [hfile-prefetch-1731594278087 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-14T14:24:39,085 DEBUG [hfile-prefetch-1731594278087 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/family/8d40b3587525458c9ba7ef76f4205c92, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-14T14:24:39,085 DEBUG [hfile-prefetch-1731594278087 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/family/8d40b3587525458c9ba7ef76f4205c92. Will sleep 100ms and try the verification again. 2024-11-14T14:24:39,185 DEBUG [hfile-prefetch-1731594278087 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/family/8d40b3587525458c9ba7ef76f4205c92, with total blocks 2, and data blocks 1 2024-11-14T14:24:39,185 DEBUG [hfile-prefetch-1731594278087 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-14T14:24:39,186 DEBUG [hfile-prefetch-1731594278087 {}] bucket.BucketCache(2294): found block for file 8d40b3587525458c9ba7ef76f4205c92 in the backing map. Acquiring read lock for offset 0 2024-11-14T14:24:39,186 DEBUG [hfile-prefetch-1731594278087 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/family/8d40b3587525458c9ba7ef76f4205c92 has now been fully cached. 2024-11-14T14:24:39,186 DEBUG [hfile-prefetch-1731594278087 {}] hfile.PrefetchExecutor(121): Prefetch completed for 8d40b3587525458c9ba7ef76f4205c92 2024-11-14T14:24:39,508 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-14T14:24:39,508 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-14T14:24:39,509 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T14:24:39,509 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-14T14:24:39,529 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-14T14:24:39,529 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731594279508 2024-11-14T14:24:39,529 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-14T14:24:40,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:24:40,110 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:24:40,113 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;6e2cc31459a7:39749,5,FailOnTimeoutGroup] 2024-11-14T14:24:40,113 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6e2cc31459a7,39749,1731594274426' ***** 2024-11-14T14:24:40,114 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-14T14:24:40,114 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-14T14:24:40,114 INFO [RS:0;6e2cc31459a7:39749 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T14:24:40,114 INFO [RS:0;6e2cc31459a7:39749 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bucket.persistence 2024-11-14T14:24:40,114 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T14:24:40,114 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T14:24:40,114 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T14:24:40,115 INFO [RS:0;6e2cc31459a7:39749 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T14:24:40,115 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T14:24:40,125 DEBUG [RS:0;6e2cc31459a7:39749 {}] bucket.BucketCache(1488): Thread RS:0;6e2cc31459a7:39749 started persisting bucket cache to file 2024-11-14T14:24:40,125 DEBUG [RS:0;6e2cc31459a7:39749 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T14:24:40,125 DEBUG [RS:0;6e2cc31459a7:39749 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-14T14:24:40,129 DEBUG [RS:0;6e2cc31459a7:39749 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-14T14:24:40,129 DEBUG [RS:0;6e2cc31459a7:39749 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731594280125 2024-11-14T14:24:40,129 DEBUG [RS:0;6e2cc31459a7:39749 {}] bucket.BucketCache(1510): Thread RS:0;6e2cc31459a7:39749 finished persisting bucket cache to file, renaming 2024-11-14T14:24:40,129 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:24:40,130 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:24:40,130 INFO [RS:0;6e2cc31459a7:39749 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:24:40,130 INFO [RS:0;6e2cc31459a7:39749 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:24:40,130 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(3091): Received CLOSE for 550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:40,131 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(959): stopping server 6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:40,131 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:24:40,131 INFO [RS_CLOSE_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T14:24:40,131 INFO [RS:0;6e2cc31459a7:39749 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6e2cc31459a7:39749. 2024-11-14T14:24:40,132 DEBUG [RS:0;6e2cc31459a7:39749 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:24:40,132 DEBUG [RS_CLOSE_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 550c53a10099b13fa7c462d5a83e9804, disabling compactions & flushes 2024-11-14T14:24:40,132 INFO [RS_CLOSE_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:40,132 DEBUG [RS_CLOSE_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:40,132 DEBUG [RS_CLOSE_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. after waiting 0 ms 2024-11-14T14:24:40,132 DEBUG [RS_CLOSE_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:40,138 DEBUG [RS:0;6e2cc31459a7:39749 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:24:40,139 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:24:40,139 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:24:40,139 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:24:40,139 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T14:24:40,140 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T14:24:40,140 DEBUG [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 550c53a10099b13fa7c462d5a83e9804=table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804.} 2024-11-14T14:24:40,140 INFO [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T14:24:40,140 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:24:40,140 INFO [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:24:40,141 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:24:40,141 DEBUG [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 550c53a10099b13fa7c462d5a83e9804 2024-11-14T14:24:40,141 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:24:40,141 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:24:40,141 INFO [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-14T14:24:40,145 DEBUG [RS_CLOSE_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/default/table1/550c53a10099b13fa7c462d5a83e9804/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-14T14:24:40,146 INFO [RS_CLOSE_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:40,147 DEBUG [RS_CLOSE_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 550c53a10099b13fa7c462d5a83e9804: Waiting for close lock at 1731594280131Running coprocessor pre-close hooks at 1731594280132 (+1 ms)Disabling compacts and flushes for region at 1731594280132Disabling writes for close at 1731594280132Writing region close event to WAL at 1731594280136 (+4 ms)Running coprocessor post-close hooks at 1731594280145 (+9 ms)Closed at 1731594280146 (+1 ms) 2024-11-14T14:24:40,147 DEBUG [RS_CLOSE_REGION-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804. 2024-11-14T14:24:40,156 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on da0a4f09d868472bbc2aa93128060724 2024-11-14T14:24:40,175 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/.tmp/info/da0a4f09d868472bbc2aa93128060724 is 133, key is table1,,1731594277250.550c53a10099b13fa7c462d5a83e9804./info:regioninfo/1731594277662/Put/seqid=0 2024-11-14T14:24:40,176 INFO [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/.tmp/info/da0a4f09d868472bbc2aa93128060724 2024-11-14T14:24:40,186 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3138a0989ea94412aea141770cbcddba 2024-11-14T14:24:40,203 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/.tmp/ns/3138a0989ea94412aea141770cbcddba is 43, key is default/ns:d/1731594276966/Put/seqid=0 2024-11-14T14:24:40,204 INFO [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/.tmp/ns/3138a0989ea94412aea141770cbcddba 2024-11-14T14:24:40,212 INFO [regionserver/6e2cc31459a7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:24:40,220 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e42a806318724356a1375bde24150d90 2024-11-14T14:24:40,240 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/.tmp/table/e42a806318724356a1375bde24150d90 is 42, key is table1/table:state/1731594277673/Put/seqid=0 2024-11-14T14:24:40,241 INFO [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/.tmp/table/e42a806318724356a1375bde24150d90 2024-11-14T14:24:40,245 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/.tmp/info/da0a4f09d868472bbc2aa93128060724 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/info/da0a4f09d868472bbc2aa93128060724 2024-11-14T14:24:40,247 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/info/da0a4f09d868472bbc2aa93128060724, delay=986 ms 2024-11-14T14:24:40,249 INFO [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/info/da0a4f09d868472bbc2aa93128060724, entries=10, sequenceid=11, filesize=6.3 K 2024-11-14T14:24:40,250 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/.tmp/ns/3138a0989ea94412aea141770cbcddba as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/ns/3138a0989ea94412aea141770cbcddba 2024-11-14T14:24:40,253 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/ns/3138a0989ea94412aea141770cbcddba, delay=993 ms 2024-11-14T14:24:40,255 INFO [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/ns/3138a0989ea94412aea141770cbcddba, entries=2, sequenceid=11, filesize=5.0 K 2024-11-14T14:24:40,256 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/.tmp/table/e42a806318724356a1375bde24150d90 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/table/e42a806318724356a1375bde24150d90 2024-11-14T14:24:40,258 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/table/e42a806318724356a1375bde24150d90, delay=994 ms 2024-11-14T14:24:40,260 INFO [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/table/e42a806318724356a1375bde24150d90, entries=2, sequenceid=11, filesize=5.1 K 2024-11-14T14:24:40,261 INFO [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=11, compaction requested=false 2024-11-14T14:24:40,261 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T14:24:40,264 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/info/da0a4f09d868472bbc2aa93128060724 2024-11-14T14:24:40,265 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/ns/3138a0989ea94412aea141770cbcddba 2024-11-14T14:24:40,266 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/table/e42a806318724356a1375bde24150d90 2024-11-14T14:24:40,272 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-14T14:24:40,272 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:24:40,272 INFO [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:24:40,273 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731594280140Running coprocessor pre-close hooks at 1731594280140Disabling compacts and flushes for region at 1731594280140Disabling writes for close at 1731594280141 (+1 ms)Obtaining lock to block concurrent updates at 1731594280141Preparing flush snapshotting stores in 1588230740 at 1731594280141Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1731594280142 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731594280145 (+3 ms)Flushing 1588230740/info: creating writer at 1731594280146 (+1 ms)Flushing 1588230740/info: appending metadata at 1731594280174 (+28 ms)Flushing 1588230740/info: closing flushed file at 1731594280174Flushing 1588230740/ns: creating writer at 1731594280180 (+6 ms)Flushing 1588230740/ns: appending metadata at 1731594280202 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1731594280202Flushing 1588230740/table: creating writer at 1731594280209 (+7 ms)Flushing 1588230740/table: appending metadata at 1731594280239 (+30 ms)Flushing 1588230740/table: closing flushed file at 1731594280239Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@208fd05f: reopening flushed file at 1731594280244 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fad5e96: reopening flushed file at 1731594280250 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18e37a5e: reopening flushed file at 1731594280256 (+6 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=11, compaction requested=false at 1731594280261 (+5 ms)Writing region close event to WAL at 1731594280266 (+5 ms)Running coprocessor post-close hooks at 1731594280272 (+6 ms)Closed at 1731594280272 2024-11-14T14:24:40,273 DEBUG [RS_CLOSE_META-regionserver/6e2cc31459a7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T14:24:40,341 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(976): stopping server 6e2cc31459a7,39749,1731594274426; all regions closed. 2024-11-14T14:24:40,347 DEBUG [RS:0;6e2cc31459a7:39749 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/oldWALs 2024-11-14T14:24:40,347 INFO [RS:0;6e2cc31459a7:39749 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6e2cc31459a7%2C39749%2C1731594274426.meta:.meta(num 1731594276673) 2024-11-14T14:24:40,350 DEBUG [RS:0;6e2cc31459a7:39749 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/oldWALs 2024-11-14T14:24:40,350 INFO [RS:0;6e2cc31459a7:39749 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6e2cc31459a7%2C39749%2C1731594274426:(num 1731594276184) 2024-11-14T14:24:40,350 DEBUG [RS:0;6e2cc31459a7:39749 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:24:40,350 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:24:40,350 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:24:40,351 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.ChoreService(370): Chore service for: regionserver/6e2cc31459a7:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T14:24:40,351 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:24:40,351 INFO [regionserver/6e2cc31459a7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:24:40,351 INFO [RS:0;6e2cc31459a7:39749 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39749 2024-11-14T14:24:40,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:24:40,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6e2cc31459a7,39749,1731594274426 2024-11-14T14:24:40,386 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:24:40,397 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6e2cc31459a7,39749,1731594274426] 2024-11-14T14:24:40,407 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6e2cc31459a7,39749,1731594274426 already deleted, retry=false 2024-11-14T14:24:40,407 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 6e2cc31459a7,39749,1731594274426 on 6e2cc31459a7,42801,1731594273477 2024-11-14T14:24:40,411 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 6e2cc31459a7,39749,1731594274426, splitWal=true, meta=true 2024-11-14T14:24:40,414 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=7 for 6e2cc31459a7,39749,1731594274426 (carryingMeta=true) 6e2cc31459a7,39749,1731594274426/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@62633f42[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T14:24:40,414 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 6e2cc31459a7,39749,1731594274426, splitWal=true, meta=true 2024-11-14T14:24:40,415 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6e2cc31459a7,39749,1731594274426, splitWal=true, meta=true, isMeta: true 2024-11-14T14:24:40,416 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/WALs/6e2cc31459a7,39749,1731594274426-splitting 2024-11-14T14:24:40,416 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/WALs/6e2cc31459a7,39749,1731594274426-splitting dir is empty, no logs to split. 2024-11-14T14:24:40,417 INFO [PEWorker-2 {}] master.SplitWALManager(105): 6e2cc31459a7,39749,1731594274426 WAL count=0, meta=true 2024-11-14T14:24:40,419 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/WALs/6e2cc31459a7,39749,1731594274426-splitting dir is empty, no logs to split. 2024-11-14T14:24:40,419 INFO [PEWorker-2 {}] master.SplitWALManager(105): 6e2cc31459a7,39749,1731594274426 WAL count=0, meta=true 2024-11-14T14:24:40,419 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 6e2cc31459a7,39749,1731594274426 WAL splitting is done? wals=0, meta=true 2024-11-14T14:24:40,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:24:40,422 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:24:40,423 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T14:24:40,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:24:40,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39749-0x10139bbbd3e0001, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:24:40,498 INFO [RS:0;6e2cc31459a7:39749 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:24:40,498 INFO [RS:0;6e2cc31459a7:39749 {}] regionserver.HRegionServer(1031): Exiting; stopping=6e2cc31459a7,39749,1731594274426; zookeeper connection closed. 2024-11-14T14:24:40,498 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@858e747 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@858e747 2024-11-14T14:24:40,574 WARN [6e2cc31459a7:42801 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T14:24:41,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T14:24:41,114 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:24:41,115 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:24:41,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:24:41,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:24:41,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T14:24:41,116 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T14:24:41,116 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=975984163, stopped=false 2024-11-14T14:24:41,116 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6e2cc31459a7,42801,1731594273477 2024-11-14T14:24:41,116 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '6e2cc31459a7,42801,1731594273477' ***** 2024-11-14T14:24:41,116 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-14T14:24:41,117 INFO [M:0;6e2cc31459a7:42801 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:24:41,117 INFO [M:0;6e2cc31459a7:42801 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;6e2cc31459a7:42801. 2024-11-14T14:24:41,117 DEBUG [M:0;6e2cc31459a7:42801 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:24:41,117 DEBUG [M:0;6e2cc31459a7:42801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:24:41,117 INFO [M:0;6e2cc31459a7:42801 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:24:41,117 DEBUG [M:0;6e2cc31459a7:42801 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T14:24:41,117 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T14:24:41,117 DEBUG [M:0;6e2cc31459a7:42801 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T14:24:41,117 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster-HFileCleaner.large.0-1731594275821 {}] cleaner.HFileCleaner(306): Exit Thread[master/6e2cc31459a7:0:becomeActiveMaster-HFileCleaner.large.0-1731594275821,5,FailOnTimeoutGroup] 2024-11-14T14:24:41,118 INFO [M:0;6e2cc31459a7:42801 {}] hbase.ChoreService(370): Chore service for: master/6e2cc31459a7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T14:24:41,118 INFO [M:0;6e2cc31459a7:42801 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:24:41,118 DEBUG [master/6e2cc31459a7:0:becomeActiveMaster-HFileCleaner.small.0-1731594275822 {}] cleaner.HFileCleaner(306): Exit Thread[master/6e2cc31459a7:0:becomeActiveMaster-HFileCleaner.small.0-1731594275822,5,FailOnTimeoutGroup] 2024-11-14T14:24:41,118 DEBUG [M:0;6e2cc31459a7:42801 {}] master.HMaster(1795): Stopping service threads 2024-11-14T14:24:41,118 INFO [M:0;6e2cc31459a7:42801 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T14:24:41,118 INFO [M:0;6e2cc31459a7:42801 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:24:41,119 INFO [M:0;6e2cc31459a7:42801 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:24:41,119 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T14:24:41,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T14:24:41,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:24:41,130 DEBUG [M:0;6e2cc31459a7:42801 {}] zookeeper.ZKUtil(347): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T14:24:41,130 WARN [M:0;6e2cc31459a7:42801 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T14:24:41,131 INFO [M:0;6e2cc31459a7:42801 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/.lastflushedseqids 2024-11-14T14:24:41,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:24:41,141 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T14:24:41,141 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:24:41,145 INFO [M:0;6e2cc31459a7:42801 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:24:41,324 DEBUG [6e2cc31459a7:42801 {}] assignment.AssignmentManager(2447): Stopped! Dropping assign of 1 queued regions. 2024-11-14T14:24:41,324 INFO [M:0;6e2cc31459a7:42801 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T14:24:41,325 DEBUG [M:0;6e2cc31459a7:42801 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:24:41,325 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:24:41,325 DEBUG [M:0;6e2cc31459a7:42801 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:24:41,325 DEBUG [M:0;6e2cc31459a7:42801 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:24:41,325 DEBUG [M:0;6e2cc31459a7:42801 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:24:41,325 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.17 KB heapSize=30.80 KB 2024-11-14T14:24:41,334 DEBUG [M:0;6e2cc31459a7:42801 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1f6629292412497ab8fe28b203fd44d2 2024-11-14T14:24:41,348 DEBUG [M:0;6e2cc31459a7:42801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1f6629292412497ab8fe28b203fd44d2 is 82, key is hbase:meta,,1/info:regioninfo/1731594276767/Put/seqid=0 2024-11-14T14:24:41,348 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1f6629292412497ab8fe28b203fd44d2 2024-11-14T14:24:41,356 DEBUG [M:0;6e2cc31459a7:42801 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1762732c9ef640a8a8b9582d922c05b3 2024-11-14T14:24:41,371 DEBUG [M:0;6e2cc31459a7:42801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1762732c9ef640a8a8b9582d922c05b3 is 706, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731594277678/Put/seqid=0 2024-11-14T14:24:41,372 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.56 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1762732c9ef640a8a8b9582d922c05b3 2024-11-14T14:24:41,379 DEBUG [M:0;6e2cc31459a7:42801 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b1cdc540227d467ba28efca5c61a2b0b 2024-11-14T14:24:41,392 DEBUG [M:0;6e2cc31459a7:42801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b1cdc540227d467ba28efca5c61a2b0b is 58, key is 6e2cc31459a7,39749,1731594274426/rs:/1731594280415/DeleteFamily/seqid=0 2024-11-14T14:24:41,393 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b1cdc540227d467ba28efca5c61a2b0b 2024-11-14T14:24:41,395 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b1cdc540227d467ba28efca5c61a2b0b 2024-11-14T14:24:41,396 DEBUG [M:0;6e2cc31459a7:42801 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1f6629292412497ab8fe28b203fd44d2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1f6629292412497ab8fe28b203fd44d2 2024-11-14T14:24:41,397 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1f6629292412497ab8fe28b203fd44d2, entries=8, sequenceid=64, filesize=5.5 K 2024-11-14T14:24:41,398 DEBUG [M:0;6e2cc31459a7:42801 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1762732c9ef640a8a8b9582d922c05b3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1762732c9ef640a8a8b9582d922c05b3 2024-11-14T14:24:41,399 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1762732c9ef640a8a8b9582d922c05b3, entries=8, sequenceid=64, filesize=6.5 K 2024-11-14T14:24:41,399 DEBUG [M:0;6e2cc31459a7:42801 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b1cdc540227d467ba28efca5c61a2b0b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b1cdc540227d467ba28efca5c61a2b0b 2024-11-14T14:24:41,401 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b1cdc540227d467ba28efca5c61a2b0b 2024-11-14T14:24:41,401 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6dfa39cc-4cfe-c01d-0426-310f2dd6708c/bb739701-560b-9841-5c60-97714cca4bdb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b1cdc540227d467ba28efca5c61a2b0b, entries=1, sequenceid=64, filesize=5.2 K 2024-11-14T14:24:41,402 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=64, compaction requested=false 2024-11-14T14:24:41,404 INFO [M:0;6e2cc31459a7:42801 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:24:41,405 DEBUG [M:0;6e2cc31459a7:42801 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731594281325Disabling compacts and flushes for region at 1731594281325Disabling writes for close at 1731594281325Obtaining lock to block concurrent updates at 1731594281325Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731594281325Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24747, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1731594281326 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731594281326Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731594281326Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731594281347 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731594281347Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731594281351 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731594281370 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731594281370Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731594281374 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731594281391 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731594281392 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@629c8744: reopening flushed file at 1731594281396 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a7887ae: reopening flushed file at 1731594281398 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e5ec72b: reopening flushed file at 1731594281399 (+1 ms)Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=64, compaction requested=false at 1731594281402 (+3 ms)Writing region close event to WAL at 1731594281404 (+2 ms)Closed at 1731594281404 2024-11-14T14:24:41,406 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:24:41,406 INFO [M:0;6e2cc31459a7:42801 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T14:24:41,407 INFO [M:0;6e2cc31459a7:42801 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42801 2024-11-14T14:24:41,407 INFO [M:0;6e2cc31459a7:42801 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:24:41,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:24:41,549 INFO [M:0;6e2cc31459a7:42801 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:24:41,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10139bbbd3e0000, quorum=127.0.0.1:50137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:24:41,553 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T14:24:41,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T14:24:41,577 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:24:41,584 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=45 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/6e2cc31459a7:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/6e2cc31459a7:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1731594280247 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731594278087 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1731594280253 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731594280258 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/6e2cc31459a7:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 285), ProcessCount=11 (was 11), AvailableMemoryMB=4150 (was 4773)