2024-11-09 22:23:21,637 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-09 22:23:21,661 main DEBUG Took 0.022443 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 22:23:21,662 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 22:23:21,662 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 22:23:21,663 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 22:23:21,664 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,671 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 22:23:21,687 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,688 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,689 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,689 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,690 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,691 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,692 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,693 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,693 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,694 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,695 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,695 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,696 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,696 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,697 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,697 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,698 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,698 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,699 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,699 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,699 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,700 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,700 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:23:21,701 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,701 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 22:23:21,703 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:23:21,705 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 22:23:21,706 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 22:23:21,707 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 22:23:21,709 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 22:23:21,709 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 22:23:21,720 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 22:23:21,723 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 22:23:21,725 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 22:23:21,726 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 22:23:21,727 main DEBUG createAppenders(={Console}) 2024-11-09 22:23:21,727 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-09 22:23:21,728 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-09 22:23:21,728 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-09 22:23:21,729 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 22:23:21,729 main DEBUG OutputStream closed 2024-11-09 22:23:21,729 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 22:23:21,730 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 22:23:21,730 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-09 22:23:21,812 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 22:23:21,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 22:23:21,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 22:23:21,817 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 22:23:21,817 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 22:23:21,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 22:23:21,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 22:23:21,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 22:23:21,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 22:23:21,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 22:23:21,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 22:23:21,820 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 22:23:21,820 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 22:23:21,821 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 22:23:21,821 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 22:23:21,821 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 22:23:21,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 22:23:21,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 22:23:21,825 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 22:23:21,825 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-09 22:23:21,826 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 22:23:21,826 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-09T22:23:21,841 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-09 22:23:21,844 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 22:23:21,844 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T22:23:22,103 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3 2024-11-09T22:23:22,136 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=606, ProcessCount=11, AvailableMemoryMB=3965 2024-11-09T22:23:22,240 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T22:23:22,324 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T22:23:22,339 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/cluster_b8caf257-a234-ec57-b540-cc25c20572d9, deleteOnExit=true 2024-11-09T22:23:22,438 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/cluster_b8caf257-a234-ec57-b540-cc25c20572d9/zookeeper_0, clientPort=54047, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/cluster_b8caf257-a234-ec57-b540-cc25c20572d9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/cluster_b8caf257-a234-ec57-b540-cc25c20572d9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T22:23:22,454 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54047 2024-11-09T22:23:22,455 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T22:23:22,548 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e with version=8 2024-11-09T22:23:22,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/hbase-staging 2024-11-09T22:23:22,649 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T22:23:22,906 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ea9122631f8b:0 server-side Connection retries=45 2024-11-09T22:23:22,916 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T22:23:22,917 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T22:23:22,927 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T22:23:22,927 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T22:23:22,928 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T22:23:23,074 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T22:23:23,169 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T22:23:23,212 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T22:23:23,223 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T22:23:23,227 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T22:23:23,262 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 33635 (auto-detected) 2024-11-09T22:23:23,263 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-09T22:23:23,286 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34789 2024-11-09T22:23:23,310 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34789 connecting to ZooKeeper ensemble=127.0.0.1:54047 2024-11-09T22:23:23,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:347890x0, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T22:23:23,428 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34789-0x10121b250d60000 connected 2024-11-09T22:23:23,522 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T22:23:23,523 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T22:23:23,533 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T22:23:23,537 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e, hbase.cluster.distributed=false 2024-11-09T22:23:23,564 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T22:23:23,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34789 2024-11-09T22:23:23,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34789 2024-11-09T22:23:23,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34789 2024-11-09T22:23:23,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34789 2024-11-09T22:23:23,574 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34789 2024-11-09T22:23:23,655 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ea9122631f8b:0 server-side Connection retries=45 2024-11-09T22:23:23,656 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T22:23:23,657 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T22:23:23,657 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T22:23:23,657 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T22:23:23,657 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T22:23:23,660 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T22:23:23,663 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T22:23:23,664 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35973 2024-11-09T22:23:23,666 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35973 connecting to ZooKeeper ensemble=127.0.0.1:54047 2024-11-09T22:23:23,666 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T22:23:23,667 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T22:23:23,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359730x0, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T22:23:23,777 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35973-0x10121b250d60001 connected 2024-11-09T22:23:23,777 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T22:23:23,783 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T22:23:23,796 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/bucket.cache 2024-11-09T22:23:23,804 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-09T22:23:23,805 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-09T22:23:23,808 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-09T22:23:23,810 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-09T22:23:23,810 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-09T22:23:23,822 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-09T22:23:23,822 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/bucket.persistence is complete. 2024-11-09T22:23:23,824 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-09T22:23:23,835 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T22:23:23,838 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T22:23:23,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T22:23:23,846 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35973 2024-11-09T22:23:23,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35973 2024-11-09T22:23:23,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35973 2024-11-09T22:23:23,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35973 2024-11-09T22:23:23,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35973 2024-11-09T22:23:23,873 DEBUG [M:0;ea9122631f8b:34789 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ea9122631f8b:34789 2024-11-09T22:23:23,874 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ea9122631f8b,34789,1731191002729 2024-11-09T22:23:23,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T22:23:23,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T22:23:23,888 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ea9122631f8b,34789,1731191002729 2024-11-09T22:23:23,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T22:23:23,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:23,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:23,920 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T22:23:23,922 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ea9122631f8b,34789,1731191002729 from backup master directory 2024-11-09T22:23:23,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T22:23:23,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ea9122631f8b,34789,1731191002729 2024-11-09T22:23:23,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T22:23:23,936 WARN [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T22:23:23,936 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ea9122631f8b,34789,1731191002729 2024-11-09T22:23:23,939 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T22:23:23,941 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T22:23:23,951 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/hbase.id] with ID: f165a175-49de-4ac6-8a4c-304b8788f5c6 2024-11-09T22:23:23,951 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/.tmp/hbase.id 2024-11-09T22:23:23,958 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/hbase.id] 2024-11-09T22:23:24,008 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T22:23:24,013 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T22:23:24,039 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-09T22:23:24,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:24,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:24,088 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:23:24,092 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T22:23:24,143 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:23:24,150 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T22:23:24,210 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store 2024-11-09T22:23:24,238 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T22:23:24,242 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:23:24,245 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T22:23:24,245 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:23:24,246 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:23:24,249 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T22:23:24,250 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:23:24,250 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:23:24,251 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731191004245Disabling compacts and flushes for region at 1731191004245Disabling writes for close at 1731191004249 (+4 ms)Writing region close event to WAL at 1731191004250 (+1 ms)Closed at 1731191004250 2024-11-09T22:23:24,252 WARN [master/ea9122631f8b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/.initializing 2024-11-09T22:23:24,252 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/WALs/ea9122631f8b,34789,1731191002729 2024-11-09T22:23:24,274 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T22:23:24,290 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ea9122631f8b%2C34789%2C1731191002729, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/WALs/ea9122631f8b,34789,1731191002729, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/oldWALs, maxLogs=10 2024-11-09T22:23:24,340 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/WALs/ea9122631f8b,34789,1731191002729/ea9122631f8b%2C34789%2C1731191002729.1731191004295 2024-11-09T22:23:24,341 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T22:23:24,341 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:23:24,342 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:23:24,345 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,347 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,391 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T22:23:24,435 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:24,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:23:24,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T22:23:24,444 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:24,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:23:24,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T22:23:24,451 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:24,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:23:24,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T22:23:24,457 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:24,458 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:23:24,459 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,463 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,463 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,468 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,469 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,472 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T22:23:24,475 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:23:24,487 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T22:23:24,488 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62327890, jitterRate=-0.0712420642375946}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T22:23:24,499 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731191004363Initializing all the Stores at 1731191004363Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191004364 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191004368 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191004369 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191004369Cleaning up temporary data from old regions at 1731191004469 (+100 ms)Region opened successfully at 1731191004499 (+30 ms) 2024-11-09T22:23:24,501 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T22:23:24,541 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b543ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ea9122631f8b/172.17.0.3:0 2024-11-09T22:23:24,576 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T22:23:24,589 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T22:23:24,589 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T22:23:24,592 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T22:23:24,593 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T22:23:24,600 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-09T22:23:24,600 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T22:23:24,640 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T22:23:24,651 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T22:23:24,707 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T22:23:24,710 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T22:23:24,712 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T22:23:24,724 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T22:23:24,727 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T22:23:24,732 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T22:23:24,745 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T22:23:24,747 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T22:23:24,759 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T22:23:24,776 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T22:23:24,787 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T22:23:24,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T22:23:24,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T22:23:24,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:24,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:24,811 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ea9122631f8b,34789,1731191002729, sessionid=0x10121b250d60000, setting cluster-up flag (Was=false) 2024-11-09T22:23:24,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:24,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:24,871 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T22:23:24,874 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ea9122631f8b,34789,1731191002729 2024-11-09T22:23:24,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:24,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:24,927 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T22:23:24,930 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ea9122631f8b,34789,1731191002729 2024-11-09T22:23:24,934 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T22:23:24,967 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(746): ClusterId : f165a175-49de-4ac6-8a4c-304b8788f5c6 2024-11-09T22:23:24,969 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T22:23:24,982 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T22:23:24,983 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T22:23:24,992 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T22:23:24,993 DEBUG [RS:0;ea9122631f8b:35973 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fb5344, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ea9122631f8b/172.17.0.3:0 2024-11-09T22:23:25,011 DEBUG [RS:0;ea9122631f8b:35973 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ea9122631f8b:35973 2024-11-09T22:23:25,014 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T22:23:25,015 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T22:23:25,015 DEBUG [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T22:23:25,014 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T22:23:25,018 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(2659): reportForDuty to master=ea9122631f8b,34789,1731191002729 with port=35973, startcode=1731191003620 2024-11-09T22:23:25,026 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T22:23:25,029 DEBUG [RS:0;ea9122631f8b:35973 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T22:23:25,032 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T22:23:25,038 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ea9122631f8b,34789,1731191002729 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T22:23:25,046 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ea9122631f8b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T22:23:25,047 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ea9122631f8b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T22:23:25,047 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ea9122631f8b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T22:23:25,047 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ea9122631f8b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T22:23:25,048 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ea9122631f8b:0, corePoolSize=10, maxPoolSize=10 2024-11-09T22:23:25,048 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,048 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ea9122631f8b:0, corePoolSize=2, maxPoolSize=2 2024-11-09T22:23:25,048 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,054 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T22:23:25,055 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T22:23:25,059 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:25,059 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T22:23:25,060 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731191035060 2024-11-09T22:23:25,063 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T22:23:25,064 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T22:23:25,067 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T22:23:25,068 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T22:23:25,068 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T22:23:25,069 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T22:23:25,074 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,081 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T22:23:25,082 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T22:23:25,083 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T22:23:25,088 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T22:23:25,089 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T22:23:25,090 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ea9122631f8b:0:becomeActiveMaster-HFileCleaner.large.0-1731191005090,5,FailOnTimeoutGroup] 2024-11-09T22:23:25,091 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ea9122631f8b:0:becomeActiveMaster-HFileCleaner.small.0-1731191005090,5,FailOnTimeoutGroup] 2024-11-09T22:23:25,091 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,091 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T22:23:25,092 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,093 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,101 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T22:23:25,102 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e 2024-11-09T22:23:25,106 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50055, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T22:23:25,114 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34789 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ea9122631f8b,35973,1731191003620 2024-11-09T22:23:25,115 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:23:25,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-09T22:23:25,117 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34789 {}] master.ServerManager(517): Registering regionserver=ea9122631f8b,35973,1731191003620 2024-11-09T22:23:25,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T22:23:25,124 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:25,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:23:25,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-09T22:23:25,129 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T22:23:25,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:25,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:23:25,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-09T22:23:25,133 DEBUG [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e 2024-11-09T22:23:25,133 DEBUG [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-09T22:23:25,134 DEBUG [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T22:23:25,137 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T22:23:25,137 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:25,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:23:25,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-09T22:23:25,141 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T22:23:25,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:25,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:23:25,142 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T22:23:25,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740 2024-11-09T22:23:25,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740 2024-11-09T22:23:25,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T22:23:25,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T22:23:25,144 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T22:23:25,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T22:23:25,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T22:23:25,150 DEBUG [RS:0;ea9122631f8b:35973 {}] zookeeper.ZKUtil(111): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ea9122631f8b,35973,1731191003620 2024-11-09T22:23:25,150 WARN [RS:0;ea9122631f8b:35973 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T22:23:25,150 INFO [RS:0;ea9122631f8b:35973 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T22:23:25,150 DEBUG [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/WALs/ea9122631f8b,35973,1731191003620 2024-11-09T22:23:25,152 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ea9122631f8b,35973,1731191003620] 2024-11-09T22:23:25,156 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T22:23:25,156 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75443000, jitterRate=0.12418830394744873}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T22:23:25,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731191005115Initializing all the Stores at 1731191005116 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191005116Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191005116Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191005116Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191005116Cleaning up temporary data from old regions at 1731191005144 (+28 ms)Region opened successfully at 1731191005162 (+18 ms) 2024-11-09T22:23:25,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T22:23:25,162 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T22:23:25,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T22:23:25,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T22:23:25,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T22:23:25,165 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T22:23:25,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731191005162Disabling compacts and flushes for region at 1731191005162Disabling writes for close at 1731191005163 (+1 ms)Writing region close event to WAL at 1731191005164 (+1 ms)Closed at 1731191005165 (+1 ms) 2024-11-09T22:23:25,168 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T22:23:25,168 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T22:23:25,176 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T22:23:25,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T22:23:25,187 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T22:23:25,189 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T22:23:25,195 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T22:23:25,200 INFO [RS:0;ea9122631f8b:35973 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T22:23:25,200 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,201 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T22:23:25,206 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T22:23:25,207 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,207 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,207 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,207 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,208 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,208 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,208 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ea9122631f8b:0, corePoolSize=2, maxPoolSize=2 2024-11-09T22:23:25,208 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,208 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,209 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,209 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,209 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,209 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ea9122631f8b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:23:25,209 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ea9122631f8b:0, corePoolSize=3, maxPoolSize=3 2024-11-09T22:23:25,210 DEBUG [RS:0;ea9122631f8b:35973 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ea9122631f8b:0, corePoolSize=3, maxPoolSize=3 2024-11-09T22:23:25,212 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,212 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,212 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,213 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,213 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,213 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(168): Chore ScheduledChore name=ea9122631f8b,35973,1731191003620-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T22:23:25,233 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T22:23:25,234 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(168): Chore ScheduledChore name=ea9122631f8b,35973,1731191003620-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,235 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,235 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.Replication(171): ea9122631f8b,35973,1731191003620 started 2024-11-09T22:23:25,253 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:25,253 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(1482): Serving as ea9122631f8b,35973,1731191003620, RpcServer on ea9122631f8b/172.17.0.3:35973, sessionid=0x10121b250d60001 2024-11-09T22:23:25,254 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T22:23:25,254 DEBUG [RS:0;ea9122631f8b:35973 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ea9122631f8b,35973,1731191003620 2024-11-09T22:23:25,254 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ea9122631f8b,35973,1731191003620' 2024-11-09T22:23:25,254 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T22:23:25,255 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T22:23:25,256 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T22:23:25,256 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T22:23:25,256 DEBUG [RS:0;ea9122631f8b:35973 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ea9122631f8b,35973,1731191003620 2024-11-09T22:23:25,256 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ea9122631f8b,35973,1731191003620' 2024-11-09T22:23:25,256 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T22:23:25,257 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T22:23:25,257 DEBUG [RS:0;ea9122631f8b:35973 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T22:23:25,257 INFO [RS:0;ea9122631f8b:35973 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T22:23:25,258 INFO [RS:0;ea9122631f8b:35973 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T22:23:25,341 WARN [ea9122631f8b:34789 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-09T22:23:25,364 INFO [RS:0;ea9122631f8b:35973 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T22:23:25,366 INFO [RS:0;ea9122631f8b:35973 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ea9122631f8b%2C35973%2C1731191003620, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/WALs/ea9122631f8b,35973,1731191003620, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/oldWALs, maxLogs=55 2024-11-09T22:23:25,390 INFO [RS:0;ea9122631f8b:35973 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/WALs/ea9122631f8b,35973,1731191003620/ea9122631f8b%2C35973%2C1731191003620.1731191005369 2024-11-09T22:23:25,391 DEBUG [RS:0;ea9122631f8b:35973 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T22:23:25,593 DEBUG [ea9122631f8b:34789 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T22:23:25,606 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ea9122631f8b,35973,1731191003620 2024-11-09T22:23:25,612 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ea9122631f8b,35973,1731191003620, state=OPENING 2024-11-09T22:23:25,671 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T22:23:25,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:25,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:25,683 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T22:23:25,684 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T22:23:25,685 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T22:23:25,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ea9122631f8b,35973,1731191003620}] 2024-11-09T22:23:25,858 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T22:23:25,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60909, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T22:23:25,869 INFO [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T22:23:25,869 INFO [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T22:23:25,870 INFO [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T22:23:25,871 INFO [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ea9122631f8b%2C35973%2C1731191003620.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/WALs/ea9122631f8b,35973,1731191003620, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/oldWALs, maxLogs=55 2024-11-09T22:23:25,890 INFO [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/WALs/ea9122631f8b,35973,1731191003620/ea9122631f8b%2C35973%2C1731191003620.meta.1731191005873.meta 2024-11-09T22:23:25,891 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-09T22:23:25,891 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:23:25,893 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T22:23:25,895 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T22:23:25,899 INFO [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T22:23:25,903 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T22:23:25,903 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:23:25,904 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T22:23:25,904 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T22:23:25,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-09T22:23:25,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T22:23:25,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:25,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:23:25,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-09T22:23:25,908 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T22:23:25,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:25,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:23:25,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-09T22:23:25,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T22:23:25,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:25,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:23:25,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-09T22:23:25,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T22:23:25,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:25,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:23:25,912 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T22:23:25,913 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740 2024-11-09T22:23:25,914 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740 2024-11-09T22:23:25,914 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T22:23:25,914 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T22:23:25,914 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T22:23:25,915 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T22:23:25,916 INFO [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74959842, jitterRate=0.11698868870735168}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T22:23:25,916 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T22:23:25,917 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731191005904Writing region info on filesystem at 1731191005905 (+1 ms)Initializing all the Stores at 1731191005905Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191005905Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191005905Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191005905Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191005905Cleaning up temporary data from old regions at 1731191005914 (+9 ms)Running coprocessor post-open hooks at 1731191005916 (+2 ms)Region opened successfully at 1731191005917 (+1 ms) 2024-11-09T22:23:25,923 INFO [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731191005851 2024-11-09T22:23:25,932 DEBUG [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T22:23:25,932 INFO [RS_OPEN_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T22:23:25,933 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ea9122631f8b,35973,1731191003620 2024-11-09T22:23:25,935 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ea9122631f8b,35973,1731191003620, state=OPEN 2024-11-09T22:23:26,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T22:23:26,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T22:23:26,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T22:23:26,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T22:23:26,099 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ea9122631f8b,35973,1731191003620 2024-11-09T22:23:26,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T22:23:26,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ea9122631f8b,35973,1731191003620 in 412 msec 2024-11-09T22:23:26,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T22:23:26,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 929 msec 2024-11-09T22:23:26,110 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T22:23:26,110 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T22:23:26,134 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:23:26,136 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ea9122631f8b,35973,1731191003620, seqNum=-1] 2024-11-09T22:23:26,161 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:23:26,163 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44299, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:23:26,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2110 sec 2024-11-09T22:23:26,180 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731191006179, completionTime=-1 2024-11-09T22:23:26,183 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T22:23:26,183 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T22:23:26,211 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-09T22:23:26,211 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731191066211 2024-11-09T22:23:26,211 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731191126211 2024-11-09T22:23:26,211 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec 2024-11-09T22:23:26,214 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ea9122631f8b,34789,1731191002729-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:26,215 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ea9122631f8b,34789,1731191002729-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:26,215 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ea9122631f8b,34789,1731191002729-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:26,216 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ea9122631f8b:34789, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:26,217 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:26,217 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:26,224 DEBUG [master/ea9122631f8b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T22:23:26,248 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.312sec 2024-11-09T22:23:26,249 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T22:23:26,250 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T22:23:26,251 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T22:23:26,251 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T22:23:26,252 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T22:23:26,252 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ea9122631f8b,34789,1731191002729-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T22:23:26,253 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ea9122631f8b,34789,1731191002729-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T22:23:26,256 DEBUG [master/ea9122631f8b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T22:23:26,257 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T22:23:26,257 INFO [master/ea9122631f8b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ea9122631f8b,34789,1731191002729-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:23:26,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16cbfbf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:23:26,278 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T22:23:26,279 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T22:23:26,283 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ea9122631f8b,34789,-1 for getting cluster id 2024-11-09T22:23:26,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:23:26,294 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f165a175-49de-4ac6-8a4c-304b8788f5c6' 2024-11-09T22:23:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:23:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f165a175-49de-4ac6-8a4c-304b8788f5c6" 2024-11-09T22:23:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b68081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:23:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ea9122631f8b,34789,-1] 2024-11-09T22:23:26,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:23:26,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:23:26,317 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:23:26,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5975d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:23:26,320 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:23:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ea9122631f8b,35973,1731191003620, seqNum=-1] 2024-11-09T22:23:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:23:26,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:23:26,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ea9122631f8b,34789,1731191002729 2024-11-09T22:23:26,357 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T22:23:26,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ea9122631f8b,34789,1731191002729 2024-11-09T22:23:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@422499d2 2024-11-09T22:23:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T22:23:26,367 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T22:23:26,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34789 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:23:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34789 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-09T22:23:26,381 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T22:23:26,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34789 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-09T22:23:26,383 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:26,385 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T22:23:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34789 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T22:23:26,401 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d20026cf5f1d4734548151341d324a10, NAME => 'table1,,1731191006369.d20026cf5f1d4734548151341d324a10.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e 2024-11-09T22:23:26,408 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731191006369.d20026cf5f1d4734548151341d324a10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:23:26,408 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing d20026cf5f1d4734548151341d324a10, disabling compactions & flushes 2024-11-09T22:23:26,408 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:26,408 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:26,408 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731191006369.d20026cf5f1d4734548151341d324a10. after waiting 0 ms 2024-11-09T22:23:26,408 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:26,408 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:26,408 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for d20026cf5f1d4734548151341d324a10: Waiting for close lock at 1731191006408Disabling compacts and flushes for region at 1731191006408Disabling writes for close at 1731191006408Writing region close event to WAL at 1731191006408Closed at 1731191006408 2024-11-09T22:23:26,410 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T22:23:26,416 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731191006369.d20026cf5f1d4734548151341d324a10.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1731191006410"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731191006410"}]},"ts":"1731191006410"} 2024-11-09T22:23:26,421 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T22:23:26,423 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T22:23:26,426 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731191006423"}]},"ts":"1731191006423"} 2024-11-09T22:23:26,430 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-09T22:23:26,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=d20026cf5f1d4734548151341d324a10, ASSIGN}] 2024-11-09T22:23:26,434 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=d20026cf5f1d4734548151341d324a10, ASSIGN 2024-11-09T22:23:26,435 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=d20026cf5f1d4734548151341d324a10, ASSIGN; state=OFFLINE, location=ea9122631f8b,35973,1731191003620; forceNewPlan=false, retain=false 2024-11-09T22:23:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34789 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T22:23:26,587 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d20026cf5f1d4734548151341d324a10, regionState=OPENING, regionLocation=ea9122631f8b,35973,1731191003620 2024-11-09T22:23:26,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=d20026cf5f1d4734548151341d324a10, ASSIGN because future has completed 2024-11-09T22:23:26,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d20026cf5f1d4734548151341d324a10, server=ea9122631f8b,35973,1731191003620}] 2024-11-09T22:23:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34789 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T22:23:26,748 INFO [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:26,748 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d20026cf5f1d4734548151341d324a10, NAME => 'table1,,1731191006369.d20026cf5f1d4734548151341d324a10.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:23:26,749 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:26,749 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1731191006369.d20026cf5f1d4734548151341d324a10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:23:26,749 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:26,749 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:26,750 INFO [StoreOpener-d20026cf5f1d4734548151341d324a10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:26,753 INFO [StoreOpener-d20026cf5f1d4734548151341d324a10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d20026cf5f1d4734548151341d324a10 columnFamilyName family 2024-11-09T22:23:26,754 DEBUG [StoreOpener-d20026cf5f1d4734548151341d324a10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:23:26,754 INFO [StoreOpener-d20026cf5f1d4734548151341d324a10-1 {}] regionserver.HStore(327): Store=d20026cf5f1d4734548151341d324a10/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:23:26,754 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:26,756 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:26,756 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:26,756 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:26,756 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:26,758 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:26,763 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T22:23:26,764 INFO [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d20026cf5f1d4734548151341d324a10; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60136339, jitterRate=-0.10389871895313263}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T22:23:26,764 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:26,765 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d20026cf5f1d4734548151341d324a10: Running coprocessor pre-open hook at 1731191006749Writing region info on filesystem at 1731191006749Initializing all the Stores at 1731191006750 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191006750Cleaning up temporary data from old regions at 1731191006757 (+7 ms)Running coprocessor post-open hooks at 1731191006764 (+7 ms)Region opened successfully at 1731191006765 (+1 ms) 2024-11-09T22:23:26,766 INFO [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731191006369.d20026cf5f1d4734548151341d324a10., pid=6, masterSystemTime=1731191006745 2024-11-09T22:23:26,769 DEBUG [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:26,770 INFO [RS_OPEN_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:26,770 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d20026cf5f1d4734548151341d324a10, regionState=OPEN, openSeqNum=2, regionLocation=ea9122631f8b,35973,1731191003620 2024-11-09T22:23:26,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d20026cf5f1d4734548151341d324a10, server=ea9122631f8b,35973,1731191003620 because future has completed 2024-11-09T22:23:26,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T22:23:26,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d20026cf5f1d4734548151341d324a10, server=ea9122631f8b,35973,1731191003620 in 183 msec 2024-11-09T22:23:26,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T22:23:26,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=d20026cf5f1d4734548151341d324a10, ASSIGN in 345 msec 2024-11-09T22:23:26,780 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T22:23:26,781 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731191006780"}]},"ts":"1731191006780"} 2024-11-09T22:23:26,783 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-09T22:23:26,784 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T22:23:26,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 409 msec 2024-11-09T22:23:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34789 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T22:23:27,018 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-09T22:23:27,018 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-09T22:23:27,020 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T22:23:27,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-09T22:23:27,027 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T22:23:27,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-09T22:23:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1731191006369.d20026cf5f1d4734548151341d324a10., hostname=ea9122631f8b,35973,1731191003620, seqNum=2] 2024-11-09T22:23:27,049 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d20026cf5f1d4734548151341d324a10 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-09T22:23:27,074 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 163b70b52e344ad5b11f4a7d0e99e4c5 2024-11-09T22:23:27,109 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/.tmp/family/163b70b52e344ad5b11f4a7d0e99e4c5 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-09T22:23:27,113 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/.tmp/family/163b70b52e344ad5b11f4a7d0e99e4c5 2024-11-09T22:23:27,145 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/.tmp/family/163b70b52e344ad5b11f4a7d0e99e4c5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/family/163b70b52e344ad5b11f4a7d0e99e4c5 2024-11-09T22:23:27,147 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/family/163b70b52e344ad5b11f4a7d0e99e4c5, delay=919 ms 2024-11-09T22:23:27,149 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/family/163b70b52e344ad5b11f4a7d0e99e4c5, entries=2, sequenceid=6, filesize=4.9 K 2024-11-09T22:23:27,156 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for d20026cf5f1d4734548151341d324a10 in 106ms, sequenceid=6, compaction requested=false 2024-11-09T22:23:27,158 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-09T22:23:27,160 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d20026cf5f1d4734548151341d324a10: 2024-11-09T22:23:28,068 DEBUG [hfile-prefetch-1731191007147 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 163b70b52e344ad5b11f4a7d0e99e4c5_0. 2024-11-09T22:23:28,072 DEBUG [hfile-prefetch-1731191007147 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 163b70b52e344ad5b11f4a7d0e99e4c5_117. 2024-11-09T22:23:28,073 DEBUG [hfile-prefetch-1731191007147 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/family/163b70b52e344ad5b11f4a7d0e99e4c5, with total blocks 2, and data blocks 1 2024-11-09T22:23:28,073 DEBUG [hfile-prefetch-1731191007147 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-09T22:23:28,073 DEBUG [hfile-prefetch-1731191007147 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/family/163b70b52e344ad5b11f4a7d0e99e4c5, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-09T22:23:28,074 DEBUG [hfile-prefetch-1731191007147 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/family/163b70b52e344ad5b11f4a7d0e99e4c5. Will sleep 100ms and try the verification again. 2024-11-09T22:23:28,174 DEBUG [hfile-prefetch-1731191007147 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/family/163b70b52e344ad5b11f4a7d0e99e4c5, with total blocks 2, and data blocks 1 2024-11-09T22:23:28,174 DEBUG [hfile-prefetch-1731191007147 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-09T22:23:28,174 DEBUG [hfile-prefetch-1731191007147 {}] bucket.BucketCache(2294): found block for file 163b70b52e344ad5b11f4a7d0e99e4c5 in the backing map. Acquiring read lock for offset 0 2024-11-09T22:23:28,174 DEBUG [hfile-prefetch-1731191007147 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/family/163b70b52e344ad5b11f4a7d0e99e4c5 has now been fully cached. 2024-11-09T22:23:28,175 DEBUG [hfile-prefetch-1731191007147 {}] hfile.PrefetchExecutor(121): Prefetch completed for 163b70b52e344ad5b11f4a7d0e99e4c5 2024-11-09T22:23:28,809 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-09T22:23:28,809 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-09T22:23:28,809 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T22:23:28,809 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-09T22:23:28,830 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-09T22:23:28,830 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731191008809 2024-11-09T22:23:28,830 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-09T22:23:29,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T22:23:29,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T22:23:29,168 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;ea9122631f8b:35973,5,FailOnTimeoutGroup] 2024-11-09T22:23:29,168 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ea9122631f8b,35973,1731191003620' ***** 2024-11-09T22:23:29,168 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-09T22:23:29,169 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-09T22:23:29,169 INFO [RS:0;ea9122631f8b:35973 {}] bucket.BucketCache(1739): Disabling cache 2024-11-09T22:23:29,169 INFO [RS:0;ea9122631f8b:35973 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/bucket.persistence 2024-11-09T22:23:29,169 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-09T22:23:29,169 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-09T22:23:29,169 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-09T22:23:29,169 INFO [RS:0;ea9122631f8b:35973 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-09T22:23:29,169 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-09T22:23:29,179 DEBUG [RS:0;ea9122631f8b:35973 {}] bucket.BucketCache(1488): Thread RS:0;ea9122631f8b:35973 started persisting bucket cache to file 2024-11-09T22:23:29,180 DEBUG [RS:0;ea9122631f8b:35973 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-09T22:23:29,180 DEBUG [RS:0;ea9122631f8b:35973 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-09T22:23:29,182 DEBUG [RS:0;ea9122631f8b:35973 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-09T22:23:29,183 DEBUG [RS:0;ea9122631f8b:35973 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731191009180 2024-11-09T22:23:29,183 DEBUG [RS:0;ea9122631f8b:35973 {}] bucket.BucketCache(1510): Thread RS:0;ea9122631f8b:35973 finished persisting bucket cache to file, renaming 2024-11-09T22:23:29,217 INFO [regionserver/ea9122631f8b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T22:23:29,217 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T22:23:29,250 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T22:23:29,250 INFO [RS:0;ea9122631f8b:35973 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T22:23:29,251 INFO [RS:0;ea9122631f8b:35973 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T22:23:29,251 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(3091): Received CLOSE for d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:29,251 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(959): stopping server ea9122631f8b,35973,1731191003620 2024-11-09T22:23:29,251 INFO [RS:0;ea9122631f8b:35973 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T22:23:29,252 INFO [RS:0;ea9122631f8b:35973 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ea9122631f8b:35973. 2024-11-09T22:23:29,252 INFO [RS_CLOSE_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1731191006369.d20026cf5f1d4734548151341d324a10. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-09T22:23:29,252 DEBUG [RS:0;ea9122631f8b:35973 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:23:29,252 DEBUG [RS_CLOSE_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d20026cf5f1d4734548151341d324a10, disabling compactions & flushes 2024-11-09T22:23:29,252 INFO [RS_CLOSE_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:29,252 DEBUG [RS_CLOSE_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:29,252 DEBUG [RS_CLOSE_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731191006369.d20026cf5f1d4734548151341d324a10. after waiting 0 ms 2024-11-09T22:23:29,252 DEBUG [RS_CLOSE_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:29,257 DEBUG [RS:0;ea9122631f8b:35973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:23:29,257 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T22:23:29,257 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T22:23:29,257 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T22:23:29,258 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T22:23:29,259 INFO [regionserver/ea9122631f8b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T22:23:29,259 INFO [regionserver/ea9122631f8b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T22:23:29,260 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-09T22:23:29,260 INFO [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-09T22:23:29,260 DEBUG [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(1325): Online Regions={d20026cf5f1d4734548151341d324a10=table1,,1731191006369.d20026cf5f1d4734548151341d324a10., 1588230740=hbase:meta,,1.1588230740} 2024-11-09T22:23:29,260 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T22:23:29,260 DEBUG [RS_CLOSE_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/default/table1/d20026cf5f1d4734548151341d324a10/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-09T22:23:29,260 INFO [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T22:23:29,261 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T22:23:29,261 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T22:23:29,261 DEBUG [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d20026cf5f1d4734548151341d324a10 2024-11-09T22:23:29,261 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T22:23:29,261 INFO [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-09T22:23:29,262 INFO [RS_CLOSE_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:29,262 DEBUG [RS_CLOSE_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d20026cf5f1d4734548151341d324a10: Waiting for close lock at 1731191009251Running coprocessor pre-close hooks at 1731191009252 (+1 ms)Disabling compacts and flushes for region at 1731191009252Disabling writes for close at 1731191009252Writing region close event to WAL at 1731191009254 (+2 ms)Running coprocessor post-close hooks at 1731191009261 (+7 ms)Closed at 1731191009262 (+1 ms) 2024-11-09T22:23:29,263 DEBUG [RS_CLOSE_REGION-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731191006369.d20026cf5f1d4734548151341d324a10. 2024-11-09T22:23:29,275 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7e5b84285c154cc38dfa8cebe9b7e171 2024-11-09T22:23:29,291 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/.tmp/info/7e5b84285c154cc38dfa8cebe9b7e171 is 133, key is table1,,1731191006369.d20026cf5f1d4734548151341d324a10./info:regioninfo/1731191006770/Put/seqid=0 2024-11-09T22:23:29,292 INFO [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/.tmp/info/7e5b84285c154cc38dfa8cebe9b7e171 2024-11-09T22:23:29,301 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a7eba7fa52f542d6bea8531d5754a73a 2024-11-09T22:23:29,318 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/.tmp/ns/a7eba7fa52f542d6bea8531d5754a73a is 43, key is default/ns:d/1731191006167/Put/seqid=0 2024-11-09T22:23:29,319 INFO [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/.tmp/ns/a7eba7fa52f542d6bea8531d5754a73a 2024-11-09T22:23:29,333 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 519d3df91fbe46eeaf0dbfc164a49def 2024-11-09T22:23:29,346 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/.tmp/table/519d3df91fbe46eeaf0dbfc164a49def is 42, key is table1/table:state/1731191006780/Put/seqid=0 2024-11-09T22:23:29,346 INFO [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/.tmp/table/519d3df91fbe46eeaf0dbfc164a49def 2024-11-09T22:23:29,349 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/.tmp/info/7e5b84285c154cc38dfa8cebe9b7e171 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/info/7e5b84285c154cc38dfa8cebe9b7e171 2024-11-09T22:23:29,350 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/info/7e5b84285c154cc38dfa8cebe9b7e171, delay=911 ms 2024-11-09T22:23:29,352 INFO [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/info/7e5b84285c154cc38dfa8cebe9b7e171, entries=10, sequenceid=11, filesize=6.3 K 2024-11-09T22:23:29,353 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/.tmp/ns/a7eba7fa52f542d6bea8531d5754a73a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/ns/a7eba7fa52f542d6bea8531d5754a73a 2024-11-09T22:23:29,354 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/ns/a7eba7fa52f542d6bea8531d5754a73a, delay=901 ms 2024-11-09T22:23:29,355 INFO [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/ns/a7eba7fa52f542d6bea8531d5754a73a, entries=2, sequenceid=11, filesize=5.0 K 2024-11-09T22:23:29,356 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/.tmp/table/519d3df91fbe46eeaf0dbfc164a49def as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/table/519d3df91fbe46eeaf0dbfc164a49def 2024-11-09T22:23:29,357 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/table/519d3df91fbe46eeaf0dbfc164a49def, delay=920 ms 2024-11-09T22:23:29,358 INFO [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/table/519d3df91fbe46eeaf0dbfc164a49def, entries=2, sequenceid=11, filesize=5.1 K 2024-11-09T22:23:29,359 INFO [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 98ms, sequenceid=11, compaction requested=false 2024-11-09T22:23:29,360 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T22:23:29,362 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/info/7e5b84285c154cc38dfa8cebe9b7e171 2024-11-09T22:23:29,363 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/ns/a7eba7fa52f542d6bea8531d5754a73a 2024-11-09T22:23:29,363 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/table/519d3df91fbe46eeaf0dbfc164a49def 2024-11-09T22:23:29,370 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-09T22:23:29,371 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T22:23:29,371 INFO [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T22:23:29,372 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731191009260Running coprocessor pre-close hooks at 1731191009260Disabling compacts and flushes for region at 1731191009260Disabling writes for close at 1731191009261 (+1 ms)Obtaining lock to block concurrent updates at 1731191009261Preparing flush snapshotting stores in 1588230740 at 1731191009261Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1731191009262 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731191009262Flushing 1588230740/info: creating writer at 1731191009262Flushing 1588230740/info: appending metadata at 1731191009290 (+28 ms)Flushing 1588230740/info: closing flushed file at 1731191009290Flushing 1588230740/ns: creating writer at 1731191009294 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731191009317 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1731191009317Flushing 1588230740/table: creating writer at 1731191009327 (+10 ms)Flushing 1588230740/table: appending metadata at 1731191009345 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731191009345Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c1cc1b1: reopening flushed file at 1731191009349 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@604216d3: reopening flushed file at 1731191009352 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a1f8994: reopening flushed file at 1731191009356 (+4 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 98ms, sequenceid=11, compaction requested=false at 1731191009359 (+3 ms)Writing region close event to WAL at 1731191009364 (+5 ms)Running coprocessor post-close hooks at 1731191009371 (+7 ms)Closed at 1731191009371 2024-11-09T22:23:29,372 DEBUG [RS_CLOSE_META-regionserver/ea9122631f8b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T22:23:29,461 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(976): stopping server ea9122631f8b,35973,1731191003620; all regions closed. 2024-11-09T22:23:29,466 DEBUG [RS:0;ea9122631f8b:35973 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/oldWALs 2024-11-09T22:23:29,466 INFO [RS:0;ea9122631f8b:35973 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ea9122631f8b%2C35973%2C1731191003620.meta:.meta(num 1731191005873) 2024-11-09T22:23:29,468 DEBUG [RS:0;ea9122631f8b:35973 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/oldWALs 2024-11-09T22:23:29,468 INFO [RS:0;ea9122631f8b:35973 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ea9122631f8b%2C35973%2C1731191003620:(num 1731191005369) 2024-11-09T22:23:29,468 DEBUG [RS:0;ea9122631f8b:35973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:23:29,468 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T22:23:29,468 INFO [RS:0;ea9122631f8b:35973 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T22:23:29,469 INFO [RS:0;ea9122631f8b:35973 {}] hbase.ChoreService(370): Chore service for: regionserver/ea9122631f8b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T22:23:29,469 INFO [RS:0;ea9122631f8b:35973 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T22:23:29,469 INFO [regionserver/ea9122631f8b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T22:23:29,469 INFO [RS:0;ea9122631f8b:35973 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35973 2024-11-09T22:23:29,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ea9122631f8b,35973,1731191003620 2024-11-09T22:23:29,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T22:23:29,492 INFO [RS:0;ea9122631f8b:35973 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T22:23:29,493 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ea9122631f8b,35973,1731191003620] 2024-11-09T22:23:29,513 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ea9122631f8b,35973,1731191003620 already deleted, retry=false 2024-11-09T22:23:29,513 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ea9122631f8b,35973,1731191003620 on ea9122631f8b,34789,1731191002729 2024-11-09T22:23:29,517 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ea9122631f8b,35973,1731191003620, splitWal=true, meta=true 2024-11-09T22:23:29,519 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ea9122631f8b,35973,1731191003620, splitWal=true, meta=true 2024-11-09T22:23:29,520 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=7 for ea9122631f8b,35973,1731191003620 (carryingMeta=true) ea9122631f8b,35973,1731191003620/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3b70dd50[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-09T22:23:29,521 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ea9122631f8b,35973,1731191003620, splitWal=true, meta=true, isMeta: true 2024-11-09T22:23:29,521 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/WALs/ea9122631f8b,35973,1731191003620-splitting 2024-11-09T22:23:29,521 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/WALs/ea9122631f8b,35973,1731191003620-splitting dir is empty, no logs to split. 2024-11-09T22:23:29,521 INFO [PEWorker-2 {}] master.SplitWALManager(105): ea9122631f8b,35973,1731191003620 WAL count=0, meta=true 2024-11-09T22:23:29,523 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/WALs/ea9122631f8b,35973,1731191003620-splitting dir is empty, no logs to split. 2024-11-09T22:23:29,523 INFO [PEWorker-2 {}] master.SplitWALManager(105): ea9122631f8b,35973,1731191003620 WAL count=0, meta=true 2024-11-09T22:23:29,523 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if ea9122631f8b,35973,1731191003620 WAL splitting is done? wals=0, meta=true 2024-11-09T22:23:29,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T22:23:29,526 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T22:23:29,527 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T22:23:29,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T22:23:29,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35973-0x10121b250d60001, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T22:23:29,603 INFO [RS:0;ea9122631f8b:35973 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T22:23:29,603 INFO [RS:0;ea9122631f8b:35973 {}] regionserver.HRegionServer(1031): Exiting; stopping=ea9122631f8b,35973,1731191003620; zookeeper connection closed. 2024-11-09T22:23:29,604 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7b8ba586 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7b8ba586 2024-11-09T22:23:29,677 WARN [ea9122631f8b:34789 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-09T22:23:30,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T22:23:30,169 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T22:23:30,169 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:23:30,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:23:30,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:23:30,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:23:30,170 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T22:23:30,170 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1791327894, stopped=false 2024-11-09T22:23:30,170 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ea9122631f8b,34789,1731191002729 2024-11-09T22:23:30,170 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'ea9122631f8b,34789,1731191002729' ***** 2024-11-09T22:23:30,170 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-09T22:23:30,171 INFO [M:0;ea9122631f8b:34789 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T22:23:30,171 INFO [M:0;ea9122631f8b:34789 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;ea9122631f8b:34789. 2024-11-09T22:23:30,171 DEBUG [M:0;ea9122631f8b:34789 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:23:30,171 DEBUG [M:0;ea9122631f8b:34789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:23:30,171 INFO [M:0;ea9122631f8b:34789 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T22:23:30,171 DEBUG [M:0;ea9122631f8b:34789 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T22:23:30,171 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T22:23:30,171 DEBUG [M:0;ea9122631f8b:34789 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T22:23:30,171 DEBUG [master/ea9122631f8b:0:becomeActiveMaster-HFileCleaner.large.0-1731191005090 {}] cleaner.HFileCleaner(306): Exit Thread[master/ea9122631f8b:0:becomeActiveMaster-HFileCleaner.large.0-1731191005090,5,FailOnTimeoutGroup] 2024-11-09T22:23:30,171 DEBUG [master/ea9122631f8b:0:becomeActiveMaster-HFileCleaner.small.0-1731191005090 {}] cleaner.HFileCleaner(306): Exit Thread[master/ea9122631f8b:0:becomeActiveMaster-HFileCleaner.small.0-1731191005090,5,FailOnTimeoutGroup] 2024-11-09T22:23:30,172 INFO [M:0;ea9122631f8b:34789 {}] hbase.ChoreService(370): Chore service for: master/ea9122631f8b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T22:23:30,172 INFO [M:0;ea9122631f8b:34789 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T22:23:30,172 DEBUG [M:0;ea9122631f8b:34789 {}] master.HMaster(1795): Stopping service threads 2024-11-09T22:23:30,172 INFO [M:0;ea9122631f8b:34789 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T22:23:30,172 INFO [M:0;ea9122631f8b:34789 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T22:23:30,173 INFO [M:0;ea9122631f8b:34789 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T22:23:30,173 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T22:23:30,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T22:23:30,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:23:30,222 DEBUG [M:0;ea9122631f8b:34789 {}] zookeeper.ZKUtil(347): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T22:23:30,222 WARN [M:0;ea9122631f8b:34789 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T22:23:30,223 INFO [M:0;ea9122631f8b:34789 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/.lastflushedseqids 2024-11-09T22:23:30,232 INFO [M:0;ea9122631f8b:34789 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T22:23:30,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T22:23:30,233 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T22:23:30,233 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T22:23:30,427 DEBUG [ea9122631f8b:34789 {}] assignment.AssignmentManager(2447): Stopped! Dropping assign of 1 queued regions. 2024-11-09T22:23:30,428 INFO [M:0;ea9122631f8b:34789 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T22:23:30,428 DEBUG [M:0;ea9122631f8b:34789 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T22:23:30,428 INFO [M:0;ea9122631f8b:34789 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:23:30,428 DEBUG [M:0;ea9122631f8b:34789 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:23:30,428 DEBUG [M:0;ea9122631f8b:34789 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T22:23:30,428 DEBUG [M:0;ea9122631f8b:34789 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:23:30,428 INFO [M:0;ea9122631f8b:34789 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.18 KB heapSize=30.80 KB 2024-11-09T22:23:30,436 DEBUG [M:0;ea9122631f8b:34789 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f8423f1440414497b6b7be5299ac46ff 2024-11-09T22:23:30,453 DEBUG [M:0;ea9122631f8b:34789 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f8423f1440414497b6b7be5299ac46ff is 82, key is hbase:meta,,1/info:regioninfo/1731191005933/Put/seqid=0 2024-11-09T22:23:30,454 INFO [M:0;ea9122631f8b:34789 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f8423f1440414497b6b7be5299ac46ff 2024-11-09T22:23:30,463 DEBUG [M:0;ea9122631f8b:34789 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b87a3668505e43699e92660d10be880c 2024-11-09T22:23:30,483 DEBUG [M:0;ea9122631f8b:34789 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b87a3668505e43699e92660d10be880c is 707, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731191006786/Put/seqid=0 2024-11-09T22:23:30,484 INFO [M:0;ea9122631f8b:34789 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.57 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b87a3668505e43699e92660d10be880c 2024-11-09T22:23:30,493 DEBUG [M:0;ea9122631f8b:34789 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 03784e0e5b5146829076f11c46c6f341 2024-11-09T22:23:30,505 DEBUG [M:0;ea9122631f8b:34789 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03784e0e5b5146829076f11c46c6f341 is 58, key is ea9122631f8b,35973,1731191003620/rs:/1731191009521/DeleteFamily/seqid=0 2024-11-09T22:23:30,506 INFO [M:0;ea9122631f8b:34789 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03784e0e5b5146829076f11c46c6f341 2024-11-09T22:23:30,508 INFO [M:0;ea9122631f8b:34789 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 03784e0e5b5146829076f11c46c6f341 2024-11-09T22:23:30,509 DEBUG [M:0;ea9122631f8b:34789 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f8423f1440414497b6b7be5299ac46ff as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f8423f1440414497b6b7be5299ac46ff 2024-11-09T22:23:30,510 INFO [M:0;ea9122631f8b:34789 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f8423f1440414497b6b7be5299ac46ff, entries=8, sequenceid=64, filesize=5.5 K 2024-11-09T22:23:30,511 DEBUG [M:0;ea9122631f8b:34789 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b87a3668505e43699e92660d10be880c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b87a3668505e43699e92660d10be880c 2024-11-09T22:23:30,512 INFO [M:0;ea9122631f8b:34789 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b87a3668505e43699e92660d10be880c, entries=8, sequenceid=64, filesize=6.5 K 2024-11-09T22:23:30,512 DEBUG [M:0;ea9122631f8b:34789 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03784e0e5b5146829076f11c46c6f341 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03784e0e5b5146829076f11c46c6f341 2024-11-09T22:23:30,514 INFO [M:0;ea9122631f8b:34789 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 03784e0e5b5146829076f11c46c6f341 2024-11-09T22:23:30,514 INFO [M:0;ea9122631f8b:34789 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1f89592f-64fa-fece-6ab4-706e7e128ce3/3adefcc6-bdee-66e0-5423-889d62b1382e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03784e0e5b5146829076f11c46c6f341, entries=1, sequenceid=64, filesize=5.2 K 2024-11-09T22:23:30,515 INFO [M:0;ea9122631f8b:34789 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.18 KB/24759, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=64, compaction requested=false 2024-11-09T22:23:30,516 INFO [M:0;ea9122631f8b:34789 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:23:30,517 DEBUG [M:0;ea9122631f8b:34789 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731191010428Disabling compacts and flushes for region at 1731191010428Disabling writes for close at 1731191010428Obtaining lock to block concurrent updates at 1731191010428Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731191010428Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24759, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1731191010429 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731191010429Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731191010429Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731191010453 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731191010453Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731191010457 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731191010482 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731191010482Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731191010486 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731191010504 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731191010504Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a73e627: reopening flushed file at 1731191010508 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24367529: reopening flushed file at 1731191010510 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6708a7c3: reopening flushed file at 1731191010512 (+2 ms)Finished flush of dataSize ~24.18 KB/24759, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=64, compaction requested=false at 1731191010515 (+3 ms)Writing region close event to WAL at 1731191010516 (+1 ms)Closed at 1731191010516 2024-11-09T22:23:30,518 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T22:23:30,518 INFO [M:0;ea9122631f8b:34789 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T22:23:30,519 INFO [M:0;ea9122631f8b:34789 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34789 2024-11-09T22:23:30,519 INFO [M:0;ea9122631f8b:34789 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T22:23:30,627 INFO [M:0;ea9122631f8b:34789 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T22:23:30,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T22:23:30,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34789-0x10121b250d60000, quorum=127.0.0.1:54047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T22:23:30,631 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T22:23:30,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-09T22:23:30,660 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T22:23:30,668 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=45 (was 12) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ea9122631f8b:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731191009350 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ea9122631f8b:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ea9122631f8b:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731191007147 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731191009357 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731191009354 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=597 (was 606), ProcessCount=11 (was 11), AvailableMemoryMB=3607 (was 3965)