2024-11-19 00:13:45,751 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-19 00:13:45,783 main DEBUG Took 0.029045 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 00:13:45,784 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 00:13:45,784 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 00:13:45,785 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 00:13:45,786 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,794 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 00:13:45,813 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,817 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,817 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,818 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,819 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,821 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,822 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,824 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,825 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,826 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,826 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,827 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,828 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,828 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,829 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,830 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,831 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:45,832 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,833 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 00:13:45,834 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:45,836 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 00:13:45,839 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 00:13:45,840 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 00:13:45,842 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 00:13:45,842 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 00:13:45,853 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 00:13:45,857 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 00:13:45,860 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 00:13:45,860 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 00:13:45,861 main DEBUG createAppenders(={Console}) 2024-11-19 00:13:45,862 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-19 00:13:45,863 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-19 00:13:45,863 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-19 00:13:45,864 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 00:13:45,865 main DEBUG OutputStream closed 2024-11-19 00:13:45,865 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 00:13:45,866 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 00:13:45,866 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-19 00:13:45,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 00:13:45,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 00:13:45,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 00:13:45,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 00:13:45,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 00:13:45,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 00:13:45,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 00:13:45,964 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 00:13:45,964 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 00:13:45,964 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 00:13:45,965 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 00:13:45,965 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 00:13:45,965 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 00:13:45,966 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 00:13:45,966 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 00:13:45,966 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 00:13:45,966 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 00:13:45,967 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 00:13:45,970 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 00:13:45,970 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-19 00:13:45,970 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 00:13:45,971 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-19T00:13:45,986 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-19 00:13:45,989 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 00:13:45,990 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T00:13:46,227 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c 2024-11-19T00:13:46,255 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=367, ProcessCount=11, AvailableMemoryMB=2716 2024-11-19T00:13:46,344 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T00:13:46,435 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T00:13:46,449 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/cluster_ccf0d88a-c7ee-4ed3-8d09-c03ee3d73d22, deleteOnExit=true 2024-11-19T00:13:46,555 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/cluster_ccf0d88a-c7ee-4ed3-8d09-c03ee3d73d22/zookeeper_0, clientPort=52475, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/cluster_ccf0d88a-c7ee-4ed3-8d09-c03ee3d73d22/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/cluster_ccf0d88a-c7ee-4ed3-8d09-c03ee3d73d22/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T00:13:46,567 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52475 2024-11-19T00:13:46,568 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T00:13:46,644 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be with version=8 2024-11-19T00:13:46,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/hbase-staging 2024-11-19T00:13:46,781 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T00:13:47,074 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6b1531871d35:0 server-side Connection retries=45 2024-11-19T00:13:47,084 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:47,086 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:47,094 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T00:13:47,094 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:47,095 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T00:13:47,251 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T00:13:47,334 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T00:13:47,375 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T00:13:47,385 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T00:13:47,389 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T00:13:47,430 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7331 (auto-detected) 2024-11-19T00:13:47,432 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T00:13:47,458 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44769 2024-11-19T00:13:47,482 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44769 connecting to ZooKeeper ensemble=127.0.0.1:52475 2024-11-19T00:13:47,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:447690x0, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T00:13:47,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44769-0x1005546a3620000 connected 2024-11-19T00:13:47,691 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T00:13:47,693 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T00:13:47,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T00:13:47,708 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be, hbase.cluster.distributed=false 2024-11-19T00:13:47,735 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T00:13:47,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44769 2024-11-19T00:13:47,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44769 2024-11-19T00:13:47,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44769 2024-11-19T00:13:47,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44769 2024-11-19T00:13:47,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44769 2024-11-19T00:13:47,845 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6b1531871d35:0 server-side Connection retries=45 2024-11-19T00:13:47,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:47,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:47,848 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T00:13:47,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:47,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T00:13:47,851 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T00:13:47,856 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T00:13:47,857 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39401 2024-11-19T00:13:47,860 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39401 connecting to ZooKeeper ensemble=127.0.0.1:52475 2024-11-19T00:13:47,861 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T00:13:47,862 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T00:13:47,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394010x0, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T00:13:47,876 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:394010x0, quorum=127.0.0.1:52475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T00:13:47,876 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39401-0x1005546a3620001 connected 2024-11-19T00:13:47,882 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T00:13:47,895 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/bucket.cache 2024-11-19T00:13:47,901 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T00:13:47,903 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T00:13:47,904 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-19T00:13:47,906 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T00:13:47,907 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T00:13:47,912 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T00:13:47,918 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T00:13:47,919 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T00:13:47,919 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/bucket.persistence is complete. 2024-11-19T00:13:47,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T00:13:47,927 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T00:13:47,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39401 2024-11-19T00:13:47,931 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39401 2024-11-19T00:13:47,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39401 2024-11-19T00:13:47,934 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39401 2024-11-19T00:13:47,935 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39401 2024-11-19T00:13:47,953 DEBUG [M:0;6b1531871d35:44769 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6b1531871d35:44769 2024-11-19T00:13:47,955 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6b1531871d35,44769,1731975226871 2024-11-19T00:13:47,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T00:13:47,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T00:13:47,974 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6b1531871d35,44769,1731975226871 2024-11-19T00:13:48,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:48,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T00:13:48,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:48,018 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T00:13:48,019 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6b1531871d35,44769,1731975226871 from backup master directory 2024-11-19T00:13:48,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T00:13:48,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6b1531871d35,44769,1731975226871 2024-11-19T00:13:48,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T00:13:48,030 WARN [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T00:13:48,031 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6b1531871d35,44769,1731975226871 2024-11-19T00:13:48,034 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T00:13:48,036 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T00:13:48,046 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/hbase.id] with ID: 6a06a6b7-503c-4202-af26-26bd2f84b309 2024-11-19T00:13:48,047 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/.tmp/hbase.id 2024-11-19T00:13:48,053 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/hbase.id] 2024-11-19T00:13:48,095 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T00:13:48,101 INFO [master/6b1531871d35:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T00:13:48,123 INFO [master/6b1531871d35:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-19T00:13:48,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:48,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:48,176 INFO [master/6b1531871d35:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:13:48,181 INFO [master/6b1531871d35:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T00:13:48,252 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:48,261 INFO [master/6b1531871d35:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T00:13:48,330 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store 2024-11-19T00:13:48,356 INFO [master/6b1531871d35:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T00:13:48,361 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:48,364 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T00:13:48,364 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:48,364 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:48,367 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T00:13:48,367 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:48,368 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:48,369 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731975228363Disabling compacts and flushes for region at 1731975228363Disabling writes for close at 1731975228367 (+4 ms)Writing region close event to WAL at 1731975228368 (+1 ms)Closed at 1731975228368 2024-11-19T00:13:48,370 WARN [master/6b1531871d35:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/.initializing 2024-11-19T00:13:48,371 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/WALs/6b1531871d35,44769,1731975226871 2024-11-19T00:13:48,387 INFO [master/6b1531871d35:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T00:13:48,402 INFO [master/6b1531871d35:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6b1531871d35%2C44769%2C1731975226871, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/WALs/6b1531871d35,44769,1731975226871, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/oldWALs, maxLogs=10 2024-11-19T00:13:48,439 INFO [master/6b1531871d35:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/WALs/6b1531871d35,44769,1731975226871/6b1531871d35%2C44769%2C1731975226871.1731975228406 2024-11-19T00:13:48,440 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T00:13:48,441 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:13:48,441 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:48,445 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,446 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,524 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T00:13:48,528 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:48,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:48,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,535 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T00:13:48,536 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:48,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:48,537 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,542 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T00:13:48,543 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:48,543 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:48,544 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,549 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T00:13:48,549 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:48,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:48,551 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,555 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,556 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,560 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,561 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,565 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T00:13:48,568 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:48,575 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T00:13:48,576 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66998826, jitterRate=-0.0016396939754486084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T00:13:48,588 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731975228459Initializing all the Stores at 1731975228459Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975228460 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975228460Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975228461 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975228461Cleaning up temporary data from old regions at 1731975228562 (+101 ms)Region opened successfully at 1731975228587 (+25 ms) 2024-11-19T00:13:48,590 INFO [master/6b1531871d35:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T00:13:48,631 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@757cbb71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6b1531871d35/172.17.0.2:0 2024-11-19T00:13:48,673 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T00:13:48,689 INFO [master/6b1531871d35:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T00:13:48,690 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T00:13:48,694 INFO [master/6b1531871d35:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T00:13:48,695 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T00:13:48,703 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-19T00:13:48,704 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T00:13:48,732 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T00:13:48,742 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T00:13:48,779 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T00:13:48,782 INFO [master/6b1531871d35:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T00:13:48,784 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T00:13:48,795 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T00:13:48,798 INFO [master/6b1531871d35:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T00:13:48,803 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T00:13:48,812 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T00:13:48,815 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T00:13:48,825 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T00:13:48,850 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T00:13:48,858 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T00:13:48,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T00:13:48,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T00:13:48,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:48,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:48,882 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6b1531871d35,44769,1731975226871, sessionid=0x1005546a3620000, setting cluster-up flag (Was=false) 2024-11-19T00:13:48,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:48,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:48,933 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T00:13:48,936 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6b1531871d35,44769,1731975226871 2024-11-19T00:13:48,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:48,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:48,992 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T00:13:48,998 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6b1531871d35,44769,1731975226871 2024-11-19T00:13:49,007 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T00:13:49,041 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(746): ClusterId : 6a06a6b7-503c-4202-af26-26bd2f84b309 2024-11-19T00:13:49,044 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T00:13:49,056 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T00:13:49,057 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T00:13:49,068 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T00:13:49,069 DEBUG [RS:0;6b1531871d35:39401 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d78188f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6b1531871d35/172.17.0.2:0 2024-11-19T00:13:49,088 DEBUG [RS:0;6b1531871d35:39401 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6b1531871d35:39401 2024-11-19T00:13:49,088 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T00:13:49,091 INFO [RS:0;6b1531871d35:39401 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T00:13:49,091 INFO [RS:0;6b1531871d35:39401 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T00:13:49,091 DEBUG [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T00:13:49,094 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(2659): reportForDuty to master=6b1531871d35,44769,1731975226871 with port=39401, startcode=1731975227804 2024-11-19T00:13:49,100 INFO [master/6b1531871d35:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T00:13:49,105 DEBUG [RS:0;6b1531871d35:39401 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T00:13:49,107 INFO [master/6b1531871d35:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T00:13:49,113 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6b1531871d35,44769,1731975226871 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T00:13:49,121 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6b1531871d35:0, corePoolSize=5, maxPoolSize=5 2024-11-19T00:13:49,122 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6b1531871d35:0, corePoolSize=5, maxPoolSize=5 2024-11-19T00:13:49,122 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6b1531871d35:0, corePoolSize=5, maxPoolSize=5 2024-11-19T00:13:49,122 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6b1531871d35:0, corePoolSize=5, maxPoolSize=5 2024-11-19T00:13:49,123 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6b1531871d35:0, corePoolSize=10, maxPoolSize=10 2024-11-19T00:13:49,123 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,123 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6b1531871d35:0, corePoolSize=2, maxPoolSize=2 2024-11-19T00:13:49,124 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,127 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731975259127 2024-11-19T00:13:49,127 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T00:13:49,128 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T00:13:49,130 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T00:13:49,131 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T00:13:49,132 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:49,132 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T00:13:49,134 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T00:13:49,135 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T00:13:49,135 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T00:13:49,135 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T00:13:49,136 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,141 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T00:13:49,142 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T00:13:49,142 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T00:13:49,145 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T00:13:49,147 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T00:13:49,152 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T00:13:49,153 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be 2024-11-19T00:13:49,154 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.large.0-1731975229151,5,FailOnTimeoutGroup] 2024-11-19T00:13:49,156 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.small.0-1731975229154,5,FailOnTimeoutGroup] 2024-11-19T00:13:49,156 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,157 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T00:13:49,158 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,158 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:49,167 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T00:13:49,172 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T00:13:49,172 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:49,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:49,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T00:13:49,179 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T00:13:49,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:49,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:49,180 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50157, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T00:13:49,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T00:13:49,187 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T00:13:49,187 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:49,188 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:49,188 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T00:13:49,189 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44769 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6b1531871d35,39401,1731975227804 2024-11-19T00:13:49,192 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44769 {}] master.ServerManager(517): Registering regionserver=6b1531871d35,39401,1731975227804 2024-11-19T00:13:49,193 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T00:13:49,193 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:49,194 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:49,194 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T00:13:49,196 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740 2024-11-19T00:13:49,196 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740 2024-11-19T00:13:49,197 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T00:13:49,197 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T00:13:49,198 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T00:13:49,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T00:13:49,211 DEBUG [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be 2024-11-19T00:13:49,211 DEBUG [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T00:13:49,212 DEBUG [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T00:13:49,214 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T00:13:49,215 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71827882, jitterRate=0.07031884789466858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T00:13:49,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731975229166Initializing all the Stores at 1731975229167 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975229167Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975229167Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975229167Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975229167Cleaning up temporary data from old regions at 1731975229198 (+31 ms)Region opened successfully at 1731975229221 (+23 ms) 2024-11-19T00:13:49,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T00:13:49,222 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T00:13:49,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T00:13:49,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T00:13:49,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T00:13:49,224 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T00:13:49,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731975229221Disabling compacts and flushes for region at 1731975229221Disabling writes for close at 1731975229222 (+1 ms)Writing region close event to WAL at 1731975229223 (+1 ms)Closed at 1731975229224 (+1 ms) 2024-11-19T00:13:49,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T00:13:49,226 DEBUG [RS:0;6b1531871d35:39401 {}] zookeeper.ZKUtil(111): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6b1531871d35,39401,1731975227804 2024-11-19T00:13:49,226 WARN [RS:0;6b1531871d35:39401 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T00:13:49,226 INFO [RS:0;6b1531871d35:39401 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T00:13:49,227 DEBUG [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/WALs/6b1531871d35,39401,1731975227804 2024-11-19T00:13:49,228 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T00:13:49,228 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T00:13:49,228 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6b1531871d35,39401,1731975227804] 2024-11-19T00:13:49,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T00:13:49,247 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T00:13:49,250 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T00:13:49,252 INFO [RS:0;6b1531871d35:39401 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T00:13:49,274 INFO [RS:0;6b1531871d35:39401 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T00:13:49,281 INFO [RS:0;6b1531871d35:39401 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T00:13:49,281 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,282 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T00:13:49,289 INFO [RS:0;6b1531871d35:39401 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T00:13:49,292 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,292 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,292 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,293 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,293 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,293 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,293 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6b1531871d35:0, corePoolSize=2, maxPoolSize=2 2024-11-19T00:13:49,294 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,294 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,294 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,294 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,295 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,295 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:49,295 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6b1531871d35:0, corePoolSize=3, maxPoolSize=3 2024-11-19T00:13:49,295 DEBUG [RS:0;6b1531871d35:39401 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0, corePoolSize=3, maxPoolSize=3 2024-11-19T00:13:49,296 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,297 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,297 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,297 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,297 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,297 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,39401,1731975227804-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T00:13:49,328 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T00:13:49,331 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,39401,1731975227804-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,331 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,332 INFO [RS:0;6b1531871d35:39401 {}] regionserver.Replication(171): 6b1531871d35,39401,1731975227804 started 2024-11-19T00:13:49,357 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:49,357 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(1482): Serving as 6b1531871d35,39401,1731975227804, RpcServer on 6b1531871d35/172.17.0.2:39401, sessionid=0x1005546a3620001 2024-11-19T00:13:49,358 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T00:13:49,359 DEBUG [RS:0;6b1531871d35:39401 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6b1531871d35,39401,1731975227804 2024-11-19T00:13:49,359 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6b1531871d35,39401,1731975227804' 2024-11-19T00:13:49,359 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T00:13:49,361 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T00:13:49,362 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T00:13:49,362 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T00:13:49,362 DEBUG [RS:0;6b1531871d35:39401 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6b1531871d35,39401,1731975227804 2024-11-19T00:13:49,362 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6b1531871d35,39401,1731975227804' 2024-11-19T00:13:49,362 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T00:13:49,363 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T00:13:49,364 DEBUG [RS:0;6b1531871d35:39401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T00:13:49,364 INFO [RS:0;6b1531871d35:39401 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T00:13:49,364 INFO [RS:0;6b1531871d35:39401 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T00:13:49,401 WARN [6b1531871d35:44769 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-19T00:13:49,471 INFO [RS:0;6b1531871d35:39401 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T00:13:49,473 INFO [RS:0;6b1531871d35:39401 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6b1531871d35%2C39401%2C1731975227804, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/WALs/6b1531871d35,39401,1731975227804, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/oldWALs, maxLogs=55 2024-11-19T00:13:49,494 INFO [RS:0;6b1531871d35:39401 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/WALs/6b1531871d35,39401,1731975227804/6b1531871d35%2C39401%2C1731975227804.1731975229477 2024-11-19T00:13:49,495 DEBUG [RS:0;6b1531871d35:39401 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T00:13:49,654 DEBUG [6b1531871d35:44769 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T00:13:49,668 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6b1531871d35,39401,1731975227804 2024-11-19T00:13:49,674 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6b1531871d35,39401,1731975227804, state=OPENING 2024-11-19T00:13:49,712 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T00:13:49,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:49,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:49,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T00:13:49,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T00:13:49,723 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T00:13:49,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6b1531871d35,39401,1731975227804}] 2024-11-19T00:13:49,904 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T00:13:49,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33133, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T00:13:49,917 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T00:13:49,918 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T00:13:49,918 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T00:13:49,920 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6b1531871d35%2C39401%2C1731975227804.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/WALs/6b1531871d35,39401,1731975227804, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/oldWALs, maxLogs=55 2024-11-19T00:13:49,944 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/WALs/6b1531871d35,39401,1731975227804/6b1531871d35%2C39401%2C1731975227804.meta.1731975229921.meta 2024-11-19T00:13:49,945 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T00:13:49,945 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:13:49,947 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T00:13:49,950 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T00:13:49,955 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T00:13:49,959 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T00:13:49,960 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:49,960 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T00:13:49,960 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T00:13:49,962 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T00:13:49,963 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T00:13:49,964 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:49,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:49,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T00:13:49,965 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T00:13:49,965 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:49,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:49,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T00:13:49,966 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T00:13:49,966 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:49,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:49,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T00:13:49,967 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T00:13:49,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:49,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:49,968 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T00:13:49,969 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740 2024-11-19T00:13:49,970 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740 2024-11-19T00:13:49,970 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T00:13:49,970 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T00:13:49,971 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T00:13:49,972 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T00:13:49,972 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72550497, jitterRate=0.0810866504907608}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T00:13:49,972 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T00:13:49,974 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731975229961Writing region info on filesystem at 1731975229961Initializing all the Stores at 1731975229962 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975229962Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975229962Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975229962Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975229962Cleaning up temporary data from old regions at 1731975229970 (+8 ms)Running coprocessor post-open hooks at 1731975229973 (+3 ms)Region opened successfully at 1731975229974 (+1 ms) 2024-11-19T00:13:49,979 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731975229896 2024-11-19T00:13:49,988 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T00:13:49,989 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T00:13:49,990 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6b1531871d35,39401,1731975227804 2024-11-19T00:13:49,991 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6b1531871d35,39401,1731975227804, state=OPEN 2024-11-19T00:13:50,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T00:13:50,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T00:13:50,031 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T00:13:50,031 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T00:13:50,032 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6b1531871d35,39401,1731975227804 2024-11-19T00:13:50,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T00:13:50,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6b1531871d35,39401,1731975227804 in 307 msec 2024-11-19T00:13:50,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T00:13:50,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 802 msec 2024-11-19T00:13:50,044 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T00:13:50,044 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T00:13:50,064 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:50,065 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,39401,1731975227804, seqNum=-1] 2024-11-19T00:13:50,081 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:50,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40863, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:50,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0630 sec 2024-11-19T00:13:50,101 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731975230101, completionTime=-1 2024-11-19T00:13:50,104 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T00:13:50,104 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T00:13:50,137 INFO [master/6b1531871d35:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-19T00:13:50,137 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731975290137 2024-11-19T00:13:50,138 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731975350137 2024-11-19T00:13:50,138 INFO [master/6b1531871d35:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 33 msec 2024-11-19T00:13:50,142 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,44769,1731975226871-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:50,143 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,44769,1731975226871-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:50,143 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,44769,1731975226871-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:50,145 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6b1531871d35:44769, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:50,146 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:50,147 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:50,161 DEBUG [master/6b1531871d35:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T00:13:50,192 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.161sec 2024-11-19T00:13:50,194 INFO [master/6b1531871d35:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T00:13:50,195 INFO [master/6b1531871d35:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T00:13:50,196 INFO [master/6b1531871d35:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T00:13:50,197 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T00:13:50,197 INFO [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T00:13:50,198 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,44769,1731975226871-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T00:13:50,199 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,44769,1731975226871-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T00:13:50,205 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T00:13:50,206 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T00:13:50,207 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,44769,1731975226871-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:50,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1b595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,259 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T00:13:50,259 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T00:13:50,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,44769,-1 for getting cluster id 2024-11-19T00:13:50,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:50,292 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a06a6b7-503c-4202-af26-26bd2f84b309' 2024-11-19T00:13:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a06a6b7-503c-4202-af26-26bd2f84b309" 2024-11-19T00:13:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a81ab1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,44769,-1] 2024-11-19T00:13:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:50,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:50,328 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:50,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5a12a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,336 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,39401,1731975227804, seqNum=-1] 2024-11-19T00:13:50,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:50,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:50,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6b1531871d35,44769,1731975226871 2024-11-19T00:13:50,410 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T00:13:50,419 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(321): The fetched master address is 6b1531871d35,44769,1731975226871 2024-11-19T00:13:50,442 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ea6751a 2024-11-19T00:13:50,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T00:13:50,465 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T00:13:50,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44769 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:13:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44769 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-19T00:13:50,488 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T00:13:50,492 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:50,495 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T00:13:50,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44769 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-19T00:13:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T00:13:50,538 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed82c4e019cf583c43f3d95bb8da2867, NAME => 'table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be 2024-11-19T00:13:50,563 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:50,564 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing ed82c4e019cf583c43f3d95bb8da2867, disabling compactions & flushes 2024-11-19T00:13:50,564 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:50,564 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:50,564 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. after waiting 0 ms 2024-11-19T00:13:50,564 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:50,565 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:50,565 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed82c4e019cf583c43f3d95bb8da2867: Waiting for close lock at 1731975230563Disabling compacts and flushes for region at 1731975230563Disabling writes for close at 1731975230564 (+1 ms)Writing region close event to WAL at 1731975230565 (+1 ms)Closed at 1731975230565 2024-11-19T00:13:50,570 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T00:13:50,586 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1731975230570"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731975230570"}]},"ts":"1731975230570"} 2024-11-19T00:13:50,595 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T00:13:50,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T00:13:50,603 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975230598"}]},"ts":"1731975230598"} 2024-11-19T00:13:50,616 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-19T00:13:50,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=ed82c4e019cf583c43f3d95bb8da2867, ASSIGN}] 2024-11-19T00:13:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T00:13:50,627 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=ed82c4e019cf583c43f3d95bb8da2867, ASSIGN 2024-11-19T00:13:50,628 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=ed82c4e019cf583c43f3d95bb8da2867, ASSIGN; state=OFFLINE, location=6b1531871d35,39401,1731975227804; forceNewPlan=false, retain=false 2024-11-19T00:13:50,780 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ed82c4e019cf583c43f3d95bb8da2867, regionState=OPENING, regionLocation=6b1531871d35,39401,1731975227804 2024-11-19T00:13:50,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=ed82c4e019cf583c43f3d95bb8da2867, ASSIGN because future has completed 2024-11-19T00:13:50,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed82c4e019cf583c43f3d95bb8da2867, server=6b1531871d35,39401,1731975227804}] 2024-11-19T00:13:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T00:13:50,949 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:50,950 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ed82c4e019cf583c43f3d95bb8da2867, NAME => 'table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:13:50,951 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:50,951 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:50,954 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:50,954 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:50,957 INFO [StoreOpener-ed82c4e019cf583c43f3d95bb8da2867-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:50,962 INFO [StoreOpener-ed82c4e019cf583c43f3d95bb8da2867-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed82c4e019cf583c43f3d95bb8da2867 columnFamilyName family 2024-11-19T00:13:50,963 DEBUG [StoreOpener-ed82c4e019cf583c43f3d95bb8da2867-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:50,963 INFO [StoreOpener-ed82c4e019cf583c43f3d95bb8da2867-1 {}] regionserver.HStore(327): Store=ed82c4e019cf583c43f3d95bb8da2867/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:50,964 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:50,966 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:50,966 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:50,966 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:50,966 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:50,968 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:50,993 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T00:13:50,994 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ed82c4e019cf583c43f3d95bb8da2867; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70982423, jitterRate=0.057720527052879333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T00:13:50,996 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:51,000 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ed82c4e019cf583c43f3d95bb8da2867: Running coprocessor pre-open hook at 1731975230955Writing region info on filesystem at 1731975230955Initializing all the Stores at 1731975230955Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975230955Cleaning up temporary data from old regions at 1731975230966 (+11 ms)Running coprocessor post-open hooks at 1731975230996 (+30 ms)Region opened successfully at 1731975230998 (+2 ms) 2024-11-19T00:13:51,002 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867., pid=6, masterSystemTime=1731975230942 2024-11-19T00:13:51,009 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ed82c4e019cf583c43f3d95bb8da2867, regionState=OPEN, openSeqNum=2, regionLocation=6b1531871d35,39401,1731975227804 2024-11-19T00:13:51,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed82c4e019cf583c43f3d95bb8da2867, server=6b1531871d35,39401,1731975227804 because future has completed 2024-11-19T00:13:51,017 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:51,018 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:51,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T00:13:51,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ed82c4e019cf583c43f3d95bb8da2867, server=6b1531871d35,39401,1731975227804 in 232 msec 2024-11-19T00:13:51,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T00:13:51,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=ed82c4e019cf583c43f3d95bb8da2867, ASSIGN in 401 msec 2024-11-19T00:13:51,029 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T00:13:51,029 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975231029"}]},"ts":"1731975231029"} 2024-11-19T00:13:51,033 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-19T00:13:51,037 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T00:13:51,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 557 msec 2024-11-19T00:13:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T00:13:51,144 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-19T00:13:51,144 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-19T00:13:51,147 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T00:13:51,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-19T00:13:51,158 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T00:13:51,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-19T00:13:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867., hostname=6b1531871d35,39401,1731975227804, seqNum=2] 2024-11-19T00:13:51,195 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ed82c4e019cf583c43f3d95bb8da2867 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-19T00:13:51,239 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cb54509840bb49ea9e7d1027cd90c9ba 2024-11-19T00:13:51,296 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/.tmp/family/cb54509840bb49ea9e7d1027cd90c9ba is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-19T00:13:51,304 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/.tmp/family/cb54509840bb49ea9e7d1027cd90c9ba 2024-11-19T00:13:51,389 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/.tmp/family/cb54509840bb49ea9e7d1027cd90c9ba as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/family/cb54509840bb49ea9e7d1027cd90c9ba 2024-11-19T00:13:51,392 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/family/cb54509840bb49ea9e7d1027cd90c9ba, delay=926 ms 2024-11-19T00:13:51,420 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/family/cb54509840bb49ea9e7d1027cd90c9ba, entries=2, sequenceid=6, filesize=4.9 K 2024-11-19T00:13:51,428 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for ed82c4e019cf583c43f3d95bb8da2867 in 234ms, sequenceid=6, compaction requested=false 2024-11-19T00:13:51,430 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-19T00:13:51,433 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ed82c4e019cf583c43f3d95bb8da2867: 2024-11-19T00:13:52,333 DEBUG [hfile-prefetch-1731975231404 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key cb54509840bb49ea9e7d1027cd90c9ba_0. 2024-11-19T00:13:52,339 DEBUG [hfile-prefetch-1731975231404 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key cb54509840bb49ea9e7d1027cd90c9ba_117. 2024-11-19T00:13:52,341 DEBUG [hfile-prefetch-1731975231404 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/family/cb54509840bb49ea9e7d1027cd90c9ba, with total blocks 2, and data blocks 1 2024-11-19T00:13:52,341 DEBUG [hfile-prefetch-1731975231404 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-19T00:13:52,341 DEBUG [hfile-prefetch-1731975231404 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/family/cb54509840bb49ea9e7d1027cd90c9ba, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-19T00:13:52,342 DEBUG [hfile-prefetch-1731975231404 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/family/cb54509840bb49ea9e7d1027cd90c9ba. Will sleep 100ms and try the verification again. 2024-11-19T00:13:52,442 DEBUG [hfile-prefetch-1731975231404 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/family/cb54509840bb49ea9e7d1027cd90c9ba, with total blocks 2, and data blocks 1 2024-11-19T00:13:52,442 DEBUG [hfile-prefetch-1731975231404 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-19T00:13:52,443 DEBUG [hfile-prefetch-1731975231404 {}] bucket.BucketCache(2293): found block for file cb54509840bb49ea9e7d1027cd90c9ba in the backing map. Acquiring read lock for offset 0 2024-11-19T00:13:52,443 DEBUG [hfile-prefetch-1731975231404 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/family/cb54509840bb49ea9e7d1027cd90c9ba has now been fully cached. 2024-11-19T00:13:52,443 DEBUG [hfile-prefetch-1731975231404 {}] hfile.PrefetchExecutor(121): Prefetch completed for cb54509840bb49ea9e7d1027cd90c9ba 2024-11-19T00:13:52,905 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T00:13:52,906 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T00:13:52,906 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T00:13:52,906 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-19T00:13:52,953 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-19T00:13:52,953 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731975232906 2024-11-19T00:13:52,953 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T00:13:53,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T00:13:53,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T00:13:53,456 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;6b1531871d35:39401,5,FailOnTimeoutGroup] 2024-11-19T00:13:53,457 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6b1531871d35,39401,1731975227804' ***** 2024-11-19T00:13:53,457 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-19T00:13:53,457 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-19T00:13:53,458 INFO [RS:0;6b1531871d35:39401 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T00:13:53,458 INFO [RS:0;6b1531871d35:39401 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/bucket.persistence 2024-11-19T00:13:53,458 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T00:13:53,458 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T00:13:53,458 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T00:13:53,458 INFO [RS:0;6b1531871d35:39401 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T00:13:53,459 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T00:13:53,469 DEBUG [RS:0;6b1531871d35:39401 {}] bucket.BucketCache(1487): Thread RS:0;6b1531871d35:39401 started persisting bucket cache to file 2024-11-19T00:13:53,469 DEBUG [RS:0;6b1531871d35:39401 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T00:13:53,469 DEBUG [RS:0;6b1531871d35:39401 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-19T00:13:53,484 DEBUG [RS:0;6b1531871d35:39401 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-19T00:13:53,484 DEBUG [RS:0;6b1531871d35:39401 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731975233469 2024-11-19T00:13:53,484 DEBUG [RS:0;6b1531871d35:39401 {}] bucket.BucketCache(1509): Thread RS:0;6b1531871d35:39401 finished persisting bucket cache to file, renaming 2024-11-19T00:13:53,502 INFO [regionserver/6b1531871d35:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T00:13:53,503 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T00:13:53,509 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T00:13:53,509 INFO [RS:0;6b1531871d35:39401 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T00:13:53,510 INFO [RS:0;6b1531871d35:39401 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T00:13:53,510 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(3091): Received CLOSE for ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:53,511 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(959): stopping server 6b1531871d35,39401,1731975227804 2024-11-19T00:13:53,512 INFO [RS:0;6b1531871d35:39401 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T00:13:53,512 INFO [RS:0;6b1531871d35:39401 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6b1531871d35:39401. 2024-11-19T00:13:53,512 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T00:13:53,512 DEBUG [RS:0;6b1531871d35:39401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:13:53,512 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ed82c4e019cf583c43f3d95bb8da2867, disabling compactions & flushes 2024-11-19T00:13:53,513 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:53,513 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:53,513 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. after waiting 0 ms 2024-11-19T00:13:53,513 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:53,517 DEBUG [RS:0;6b1531871d35:39401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:53,518 INFO [RS:0;6b1531871d35:39401 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T00:13:53,518 INFO [RS:0;6b1531871d35:39401 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T00:13:53,518 INFO [RS:0;6b1531871d35:39401 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T00:13:53,518 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T00:13:53,525 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-19T00:13:53,526 DEBUG [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, ed82c4e019cf583c43f3d95bb8da2867=table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867.} 2024-11-19T00:13:53,526 DEBUG [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ed82c4e019cf583c43f3d95bb8da2867 2024-11-19T00:13:53,527 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T00:13:53,527 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T00:13:53,527 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T00:13:53,527 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T00:13:53,527 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T00:13:53,528 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T00:13:53,528 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-19T00:13:53,538 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/default/table1/ed82c4e019cf583c43f3d95bb8da2867/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-19T00:13:53,539 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:53,539 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ed82c4e019cf583c43f3d95bb8da2867: Waiting for close lock at 1731975233512Running coprocessor pre-close hooks at 1731975233512Disabling compacts and flushes for region at 1731975233512Disabling writes for close at 1731975233513 (+1 ms)Writing region close event to WAL at 1731975233526 (+13 ms)Running coprocessor post-close hooks at 1731975233539 (+13 ms)Closed at 1731975233539 2024-11-19T00:13:53,540 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867. 2024-11-19T00:13:53,578 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e864be0c117149b1bc8d428fe7cbf294 2024-11-19T00:13:53,604 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/.tmp/info/e864be0c117149b1bc8d428fe7cbf294 is 133, key is table1,,1731975230468.ed82c4e019cf583c43f3d95bb8da2867./info:regioninfo/1731975231009/Put/seqid=0 2024-11-19T00:13:53,605 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/.tmp/info/e864be0c117149b1bc8d428fe7cbf294 2024-11-19T00:13:53,629 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0d6b9ae64e8844288b873bbd5af84a09 2024-11-19T00:13:53,651 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/.tmp/ns/0d6b9ae64e8844288b873bbd5af84a09 is 43, key is default/ns:d/1731975230088/Put/seqid=0 2024-11-19T00:13:53,652 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/.tmp/ns/0d6b9ae64e8844288b873bbd5af84a09 2024-11-19T00:13:53,672 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c5a3e7b05c0b4705b162784657db883f 2024-11-19T00:13:53,692 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/.tmp/table/c5a3e7b05c0b4705b162784657db883f is 42, key is table1/table:state/1731975231029/Put/seqid=0 2024-11-19T00:13:53,694 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/.tmp/table/c5a3e7b05c0b4705b162784657db883f 2024-11-19T00:13:53,698 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/.tmp/info/e864be0c117149b1bc8d428fe7cbf294 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/info/e864be0c117149b1bc8d428fe7cbf294 2024-11-19T00:13:53,700 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/info/e864be0c117149b1bc8d428fe7cbf294, delay=918 ms 2024-11-19T00:13:53,702 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/info/e864be0c117149b1bc8d428fe7cbf294, entries=10, sequenceid=11, filesize=6.3 K 2024-11-19T00:13:53,703 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/.tmp/ns/0d6b9ae64e8844288b873bbd5af84a09 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/ns/0d6b9ae64e8844288b873bbd5af84a09 2024-11-19T00:13:53,705 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/ns/0d6b9ae64e8844288b873bbd5af84a09, delay=999 ms 2024-11-19T00:13:53,708 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/ns/0d6b9ae64e8844288b873bbd5af84a09, entries=2, sequenceid=11, filesize=5.0 K 2024-11-19T00:13:53,708 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/.tmp/table/c5a3e7b05c0b4705b162784657db883f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/table/c5a3e7b05c0b4705b162784657db883f 2024-11-19T00:13:53,710 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/table/c5a3e7b05c0b4705b162784657db883f, delay=902 ms 2024-11-19T00:13:53,716 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/table/c5a3e7b05c0b4705b162784657db883f, entries=2, sequenceid=11, filesize=5.1 K 2024-11-19T00:13:53,718 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 189ms, sequenceid=11, compaction requested=false 2024-11-19T00:13:53,718 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T00:13:53,727 DEBUG [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T00:13:53,754 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/info/e864be0c117149b1bc8d428fe7cbf294 2024-11-19T00:13:53,766 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/ns/0d6b9ae64e8844288b873bbd5af84a09 2024-11-19T00:13:53,773 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/table/c5a3e7b05c0b4705b162784657db883f 2024-11-19T00:13:53,782 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-19T00:13:53,783 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T00:13:53,783 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T00:13:53,783 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731975233527Running coprocessor pre-close hooks at 1731975233527Disabling compacts and flushes for region at 1731975233527Disabling writes for close at 1731975233528 (+1 ms)Obtaining lock to block concurrent updates at 1731975233528Preparing flush snapshotting stores in 1588230740 at 1731975233528Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1731975233529 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731975233530 (+1 ms)Flushing 1588230740/info: creating writer at 1731975233530Flushing 1588230740/info: appending metadata at 1731975233602 (+72 ms)Flushing 1588230740/info: closing flushed file at 1731975233603 (+1 ms)Flushing 1588230740/ns: creating writer at 1731975233610 (+7 ms)Flushing 1588230740/ns: appending metadata at 1731975233649 (+39 ms)Flushing 1588230740/ns: closing flushed file at 1731975233650 (+1 ms)Flushing 1588230740/table: creating writer at 1731975233656 (+6 ms)Flushing 1588230740/table: appending metadata at 1731975233692 (+36 ms)Flushing 1588230740/table: closing flushed file at 1731975233692Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c1d1226: reopening flushed file at 1731975233697 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66134d4a: reopening flushed file at 1731975233703 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3877326b: reopening flushed file at 1731975233708 (+5 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 189ms, sequenceid=11, compaction requested=false at 1731975233718 (+10 ms)Writing region close event to WAL at 1731975233774 (+56 ms)Running coprocessor post-close hooks at 1731975233783 (+9 ms)Closed at 1731975233783 2024-11-19T00:13:53,784 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T00:13:53,927 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(976): stopping server 6b1531871d35,39401,1731975227804; all regions closed. 2024-11-19T00:13:53,932 DEBUG [RS:0;6b1531871d35:39401 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/oldWALs 2024-11-19T00:13:53,933 INFO [RS:0;6b1531871d35:39401 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6b1531871d35%2C39401%2C1731975227804.meta:.meta(num 1731975229921) 2024-11-19T00:13:53,935 DEBUG [RS:0;6b1531871d35:39401 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/oldWALs 2024-11-19T00:13:53,935 INFO [RS:0;6b1531871d35:39401 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6b1531871d35%2C39401%2C1731975227804:(num 1731975229477) 2024-11-19T00:13:53,935 DEBUG [RS:0;6b1531871d35:39401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:53,936 INFO [RS:0;6b1531871d35:39401 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T00:13:53,936 INFO [RS:0;6b1531871d35:39401 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T00:13:53,936 INFO [RS:0;6b1531871d35:39401 {}] hbase.ChoreService(370): Chore service for: regionserver/6b1531871d35:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T00:13:53,936 INFO [RS:0;6b1531871d35:39401 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T00:13:53,936 INFO [regionserver/6b1531871d35:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T00:13:53,937 INFO [RS:0;6b1531871d35:39401 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39401 2024-11-19T00:13:53,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T00:13:53,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6b1531871d35,39401,1731975227804 2024-11-19T00:13:53,979 INFO [RS:0;6b1531871d35:39401 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T00:13:53,988 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6b1531871d35,39401,1731975227804] 2024-11-19T00:13:53,995 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6b1531871d35,39401,1731975227804 already deleted, retry=false 2024-11-19T00:13:53,996 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 6b1531871d35,39401,1731975227804 on 6b1531871d35,44769,1731975226871 2024-11-19T00:13:54,002 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 6b1531871d35,39401,1731975227804, splitWal=true, meta=true 2024-11-19T00:13:54,005 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=7 for 6b1531871d35,39401,1731975227804 (carryingMeta=true) 6b1531871d35,39401,1731975227804/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3e3efde7[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-19T00:13:54,005 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 6b1531871d35,39401,1731975227804, splitWal=true, meta=true 2024-11-19T00:13:54,006 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6b1531871d35,39401,1731975227804, splitWal=true, meta=true, isMeta: true 2024-11-19T00:13:54,007 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/WALs/6b1531871d35,39401,1731975227804-splitting 2024-11-19T00:13:54,007 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/WALs/6b1531871d35,39401,1731975227804-splitting dir is empty, no logs to split. 2024-11-19T00:13:54,008 INFO [PEWorker-2 {}] master.SplitWALManager(105): 6b1531871d35,39401,1731975227804 WAL count=0, meta=true 2024-11-19T00:13:54,010 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/WALs/6b1531871d35,39401,1731975227804-splitting dir is empty, no logs to split. 2024-11-19T00:13:54,010 INFO [PEWorker-2 {}] master.SplitWALManager(105): 6b1531871d35,39401,1731975227804 WAL count=0, meta=true 2024-11-19T00:13:54,011 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 6b1531871d35,39401,1731975227804 WAL splitting is done? wals=0, meta=true 2024-11-19T00:13:54,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T00:13:54,014 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T00:13:54,015 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T00:13:54,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T00:13:54,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39401-0x1005546a3620001, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T00:13:54,088 INFO [RS:0;6b1531871d35:39401 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T00:13:54,088 INFO [RS:0;6b1531871d35:39401 {}] regionserver.HRegionServer(1031): Exiting; stopping=6b1531871d35,39401,1731975227804; zookeeper connection closed. 2024-11-19T00:13:54,089 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7dec97c6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7dec97c6 2024-11-19T00:13:54,166 WARN [6b1531871d35:44769 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-19T00:13:54,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T00:13:54,458 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T00:13:54,458 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:13:54,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:54,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:54,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:13:54,460 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T00:13:54,460 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1590087585, stopped=false 2024-11-19T00:13:54,460 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6b1531871d35,44769,1731975226871 2024-11-19T00:13:54,460 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '6b1531871d35,44769,1731975226871' ***** 2024-11-19T00:13:54,460 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-19T00:13:54,461 INFO [M:0;6b1531871d35:44769 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T00:13:54,461 INFO [M:0;6b1531871d35:44769 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;6b1531871d35:44769. 2024-11-19T00:13:54,461 DEBUG [M:0;6b1531871d35:44769 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:13:54,461 DEBUG [M:0;6b1531871d35:44769 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:54,461 INFO [M:0;6b1531871d35:44769 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T00:13:54,461 DEBUG [M:0;6b1531871d35:44769 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T00:13:54,461 DEBUG [M:0;6b1531871d35:44769 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T00:13:54,461 DEBUG [master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.large.0-1731975229151 {}] cleaner.HFileCleaner(306): Exit Thread[master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.large.0-1731975229151,5,FailOnTimeoutGroup] 2024-11-19T00:13:54,462 DEBUG [master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.small.0-1731975229154 {}] cleaner.HFileCleaner(306): Exit Thread[master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.small.0-1731975229154,5,FailOnTimeoutGroup] 2024-11-19T00:13:54,462 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T00:13:54,462 INFO [M:0;6b1531871d35:44769 {}] hbase.ChoreService(370): Chore service for: master/6b1531871d35:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T00:13:54,462 INFO [M:0;6b1531871d35:44769 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T00:13:54,462 DEBUG [M:0;6b1531871d35:44769 {}] master.HMaster(1795): Stopping service threads 2024-11-19T00:13:54,462 INFO [M:0;6b1531871d35:44769 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T00:13:54,462 INFO [M:0;6b1531871d35:44769 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T00:13:54,465 INFO [M:0;6b1531871d35:44769 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T00:13:54,465 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T00:13:54,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T00:13:54,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:54,509 DEBUG [M:0;6b1531871d35:44769 {}] zookeeper.ZKUtil(347): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T00:13:54,509 WARN [M:0;6b1531871d35:44769 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T00:13:54,510 INFO [M:0;6b1531871d35:44769 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/.lastflushedseqids 2024-11-19T00:13:54,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T00:13:54,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:54,522 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T00:13:54,524 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T00:13:54,553 INFO [M:0;6b1531871d35:44769 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T00:13:54,666 DEBUG [6b1531871d35:44769 {}] assignment.AssignmentManager(2447): Stopped! Dropping assign of 1 queued regions. 2024-11-19T00:13:54,666 INFO [M:0;6b1531871d35:44769 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T00:13:54,667 DEBUG [M:0;6b1531871d35:44769 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T00:13:54,667 INFO [M:0;6b1531871d35:44769 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:54,667 DEBUG [M:0;6b1531871d35:44769 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:54,667 DEBUG [M:0;6b1531871d35:44769 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T00:13:54,667 DEBUG [M:0;6b1531871d35:44769 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:54,667 INFO [M:0;6b1531871d35:44769 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.17 KB heapSize=30.80 KB 2024-11-19T00:13:54,688 DEBUG [M:0;6b1531871d35:44769 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b72108b839eb46d3b7a755ff99ba7194 2024-11-19T00:13:54,708 DEBUG [M:0;6b1531871d35:44769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b72108b839eb46d3b7a755ff99ba7194 is 82, key is hbase:meta,,1/info:regioninfo/1731975229989/Put/seqid=0 2024-11-19T00:13:54,709 INFO [M:0;6b1531871d35:44769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b72108b839eb46d3b7a755ff99ba7194 2024-11-19T00:13:54,723 DEBUG [M:0;6b1531871d35:44769 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5a08d5be5bf149c69ab7894af7b57444 2024-11-19T00:13:54,741 DEBUG [M:0;6b1531871d35:44769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5a08d5be5bf149c69ab7894af7b57444 is 706, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731975231039/Put/seqid=0 2024-11-19T00:13:54,743 INFO [M:0;6b1531871d35:44769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.56 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5a08d5be5bf149c69ab7894af7b57444 2024-11-19T00:13:54,757 DEBUG [M:0;6b1531871d35:44769 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 24dad3c14b4c427689b07b4f66636b3d 2024-11-19T00:13:54,778 DEBUG [M:0;6b1531871d35:44769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24dad3c14b4c427689b07b4f66636b3d is 58, key is 6b1531871d35,39401,1731975227804/rs:/1731975234006/DeleteFamily/seqid=0 2024-11-19T00:13:54,780 INFO [M:0;6b1531871d35:44769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24dad3c14b4c427689b07b4f66636b3d 2024-11-19T00:13:54,783 INFO [M:0;6b1531871d35:44769 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 24dad3c14b4c427689b07b4f66636b3d 2024-11-19T00:13:54,784 DEBUG [M:0;6b1531871d35:44769 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b72108b839eb46d3b7a755ff99ba7194 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b72108b839eb46d3b7a755ff99ba7194 2024-11-19T00:13:54,787 INFO [M:0;6b1531871d35:44769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b72108b839eb46d3b7a755ff99ba7194, entries=8, sequenceid=64, filesize=5.5 K 2024-11-19T00:13:54,788 DEBUG [M:0;6b1531871d35:44769 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5a08d5be5bf149c69ab7894af7b57444 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5a08d5be5bf149c69ab7894af7b57444 2024-11-19T00:13:54,790 INFO [M:0;6b1531871d35:44769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5a08d5be5bf149c69ab7894af7b57444, entries=8, sequenceid=64, filesize=6.5 K 2024-11-19T00:13:54,791 DEBUG [M:0;6b1531871d35:44769 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24dad3c14b4c427689b07b4f66636b3d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/24dad3c14b4c427689b07b4f66636b3d 2024-11-19T00:13:54,794 INFO [M:0;6b1531871d35:44769 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 24dad3c14b4c427689b07b4f66636b3d 2024-11-19T00:13:54,794 INFO [M:0;6b1531871d35:44769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4ca23ed2-db21-3f59-2d66-9502c972181c/f103b77e-822c-c555-deff-2631e9fee1be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/24dad3c14b4c427689b07b4f66636b3d, entries=1, sequenceid=64, filesize=5.2 K 2024-11-19T00:13:54,796 INFO [M:0;6b1531871d35:44769 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 128ms, sequenceid=64, compaction requested=false 2024-11-19T00:13:54,810 INFO [M:0;6b1531871d35:44769 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:54,811 DEBUG [M:0;6b1531871d35:44769 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731975234667Disabling compacts and flushes for region at 1731975234667Disabling writes for close at 1731975234667Obtaining lock to block concurrent updates at 1731975234667Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731975234667Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24747, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1731975234668 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731975234669 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731975234669Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731975234707 (+38 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731975234708 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731975234712 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731975234741 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731975234741Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731975234745 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731975234777 (+32 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731975234777Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43d87551: reopening flushed file at 1731975234784 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d6fc3ba: reopening flushed file at 1731975234787 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@295d2e07: reopening flushed file at 1731975234791 (+4 ms)Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 128ms, sequenceid=64, compaction requested=false at 1731975234796 (+5 ms)Writing region close event to WAL at 1731975234810 (+14 ms)Closed at 1731975234810 2024-11-19T00:13:54,819 INFO [M:0;6b1531871d35:44769 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T00:13:54,820 INFO [M:0;6b1531871d35:44769 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44769 2024-11-19T00:13:54,820 INFO [M:0;6b1531871d35:44769 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T00:13:54,820 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T00:13:54,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T00:13:54,930 INFO [M:0;6b1531871d35:44769 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T00:13:54,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44769-0x1005546a3620000, quorum=127.0.0.1:52475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T00:13:54,937 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T00:13:54,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-19T00:13:54,971 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T00:13:54,993 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=44 (was 12) Potentially hanging thread: master/6b1531871d35:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731975233700 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/6b1531871d35:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/6b1531871d35:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731975233705 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.metrics2.impl.JmxCacheBuster$JmxCacheBusterRunnable.run(JmxCacheBuster.java:101) java.base@17.0.11/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731975231404 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731975233710 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=381 (was 367) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2344 (was 2716)