2024-11-24 09:55:41,782 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-24 09:55:41,799 main DEBUG Took 0.015319 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 09:55:41,799 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 09:55:41,799 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 09:55:41,800 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 09:55:41,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,808 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 09:55:41,823 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,825 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,826 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,826 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,828 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,828 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,829 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,830 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,831 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,832 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,832 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,833 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,833 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,834 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,834 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,835 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,836 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,836 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,837 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 09:55:41,837 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,838 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 09:55:41,840 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 09:55:41,841 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 09:55:41,843 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 09:55:41,844 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 09:55:41,846 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 09:55:41,846 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 09:55:41,857 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 09:55:41,860 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 09:55:41,863 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 09:55:41,863 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 09:55:41,864 main DEBUG createAppenders(={Console}) 2024-11-24 09:55:41,865 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-24 09:55:41,865 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-24 09:55:41,866 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-24 09:55:41,866 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 09:55:41,867 main DEBUG OutputStream closed 2024-11-24 09:55:41,867 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 09:55:41,867 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 09:55:41,868 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-24 09:55:41,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 09:55:41,953 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 09:55:41,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 09:55:41,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 09:55:41,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 09:55:41,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 09:55:41,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 09:55:41,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 09:55:41,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 09:55:41,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 09:55:41,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 09:55:41,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 09:55:41,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 09:55:41,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 09:55:41,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 09:55:41,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 09:55:41,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 09:55:41,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 09:55:41,962 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 09:55:41,962 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-24 09:55:41,962 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 09:55:41,963 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-24T09:55:41,975 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-24 09:55:41,977 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 09:55:41,978 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T09:55:42,191 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a 2024-11-24T09:55:42,215 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=273, ProcessCount=11, AvailableMemoryMB=8589 2024-11-24T09:55:42,301 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T09:55:42,398 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T09:55:42,414 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/cluster_93088ea4-022b-504f-a474-9b037f5aec87, deleteOnExit=true 2024-11-24T09:55:42,540 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/cluster_93088ea4-022b-504f-a474-9b037f5aec87/zookeeper_0, clientPort=51787, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/cluster_93088ea4-022b-504f-a474-9b037f5aec87/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/cluster_93088ea4-022b-504f-a474-9b037f5aec87/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T09:55:42,553 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51787 2024-11-24T09:55:42,553 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T09:55:42,645 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c with version=8 2024-11-24T09:55:42,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/hbase-staging 2024-11-24T09:55:42,744 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T09:55:42,985 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ba35ada4783f:0 server-side Connection retries=45 2024-11-24T09:55:42,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:55:42,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T09:55:43,002 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T09:55:43,002 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:55:43,003 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T09:55:43,139 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T09:55:43,215 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T09:55:43,250 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T09:55:43,260 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T09:55:43,264 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T09:55:43,299 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 96330 (auto-detected) 2024-11-24T09:55:43,300 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T09:55:43,317 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36847 2024-11-24T09:55:43,338 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36847 connecting to ZooKeeper ensemble=127.0.0.1:51787 2024-11-24T09:55:43,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:368470x0, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T09:55:43,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36847-0x1016d8ec9fe0000 connected 2024-11-24T09:55:43,535 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T09:55:43,536 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T09:55:43,544 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:55:43,548 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c, hbase.cluster.distributed=false 2024-11-24T09:55:43,572 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T09:55:43,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36847 2024-11-24T09:55:43,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36847 2024-11-24T09:55:43,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36847 2024-11-24T09:55:43,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36847 2024-11-24T09:55:43,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36847 2024-11-24T09:55:43,656 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba35ada4783f:0 server-side Connection retries=45 2024-11-24T09:55:43,657 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:55:43,658 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T09:55:43,658 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T09:55:43,658 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T09:55:43,658 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T09:55:43,661 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T09:55:43,664 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T09:55:43,665 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42519 2024-11-24T09:55:43,667 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42519 connecting to ZooKeeper ensemble=127.0.0.1:51787 2024-11-24T09:55:43,668 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T09:55:43,669 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T09:55:43,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425190x0, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T09:55:43,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:425190x0, quorum=127.0.0.1:51787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:55:43,691 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42519-0x1016d8ec9fe0001 connected 2024-11-24T09:55:43,697 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T09:55:43,710 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/bucket.cache 2024-11-24T09:55:43,718 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T09:55:43,720 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T09:55:43,722 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-24T09:55:43,723 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T09:55:43,724 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T09:55:43,730 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T09:55:43,732 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T09:55:43,733 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/bucket.persistence is complete. 2024-11-24T09:55:43,737 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T09:55:43,740 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T09:55:43,747 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T09:55:43,747 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42519 2024-11-24T09:55:43,748 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42519 2024-11-24T09:55:43,748 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42519 2024-11-24T09:55:43,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42519 2024-11-24T09:55:43,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42519 2024-11-24T09:55:43,765 DEBUG [M:0;ba35ada4783f:36847 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ba35ada4783f:36847 2024-11-24T09:55:43,766 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ba35ada4783f,36847,1732442142820 2024-11-24T09:55:43,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:55:43,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:55:43,809 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ba35ada4783f,36847,1732442142820 2024-11-24T09:55:43,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:43,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T09:55:43,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:43,841 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T09:55:43,843 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ba35ada4783f,36847,1732442142820 from backup master directory 2024-11-24T09:55:43,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:55:43,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ba35ada4783f,36847,1732442142820 2024-11-24T09:55:43,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T09:55:43,857 WARN [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T09:55:43,857 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ba35ada4783f,36847,1732442142820 2024-11-24T09:55:43,860 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T09:55:43,863 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T09:55:43,874 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/hbase.id] with ID: d4c77c4a-341a-48a8-9f7b-ddcebd83d205 2024-11-24T09:55:43,875 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/.tmp/hbase.id 2024-11-24T09:55:43,881 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/hbase.id] 2024-11-24T09:55:43,923 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T09:55:43,930 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T09:55:43,956 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-24T09:55:43,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:43,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:44,006 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T09:55:44,010 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T09:55:44,051 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T09:55:44,056 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:55:44,112 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store 2024-11-24T09:55:44,133 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T09:55:44,137 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:55:44,138 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T09:55:44,138 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:55:44,139 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:55:44,140 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T09:55:44,140 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:55:44,141 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:55:44,141 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732442144138Disabling compacts and flushes for region at 1732442144138Disabling writes for close at 1732442144140 (+2 ms)Writing region close event to WAL at 1732442144141 (+1 ms)Closed at 1732442144141 2024-11-24T09:55:44,142 WARN [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/.initializing 2024-11-24T09:55:44,142 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/WALs/ba35ada4783f,36847,1732442142820 2024-11-24T09:55:44,153 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:55:44,165 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba35ada4783f%2C36847%2C1732442142820, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/WALs/ba35ada4783f,36847,1732442142820, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/oldWALs, maxLogs=10 2024-11-24T09:55:44,199 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/WALs/ba35ada4783f,36847,1732442142820/ba35ada4783f%2C36847%2C1732442142820.1732442144169 2024-11-24T09:55:44,200 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T09:55:44,201 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:55:44,201 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:55:44,205 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,206 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,240 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,265 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T09:55:44,269 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:44,271 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:55:44,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,276 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T09:55:44,276 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:44,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:55:44,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,282 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T09:55:44,282 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:44,282 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:55:44,283 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,287 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T09:55:44,287 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:44,287 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:55:44,288 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,292 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,293 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,297 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,298 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,302 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T09:55:44,305 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T09:55:44,312 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:55:44,314 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74727496, jitterRate=0.11352646350860596}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T09:55:44,325 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732442144218Initializing all the Stores at 1732442144218Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732442144219 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732442144220 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732442144220Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732442144221 (+1 ms)Cleaning up temporary data from old regions at 1732442144298 (+77 ms)Region opened successfully at 1732442144325 (+27 ms) 2024-11-24T09:55:44,328 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T09:55:44,356 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ba66f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba35ada4783f/172.17.0.2:0 2024-11-24T09:55:44,386 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T09:55:44,399 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T09:55:44,399 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T09:55:44,402 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T09:55:44,403 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T09:55:44,408 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T09:55:44,409 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T09:55:44,435 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T09:55:44,444 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T09:55:44,473 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T09:55:44,477 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T09:55:44,479 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T09:55:44,489 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T09:55:44,493 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T09:55:44,498 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T09:55:44,506 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T09:55:44,508 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T09:55:44,514 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T09:55:44,533 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T09:55:44,539 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T09:55:44,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T09:55:44,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T09:55:44,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:44,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:44,559 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ba35ada4783f,36847,1732442142820, sessionid=0x1016d8ec9fe0000, setting cluster-up flag (Was=false) 2024-11-24T09:55:44,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:44,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:44,606 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T09:55:44,609 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba35ada4783f,36847,1732442142820 2024-11-24T09:55:44,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:44,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:44,648 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T09:55:44,651 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba35ada4783f,36847,1732442142820 2024-11-24T09:55:44,657 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T09:55:44,724 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T09:55:44,733 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T09:55:44,739 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T09:55:44,745 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ba35ada4783f,36847,1732442142820 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T09:55:44,752 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:55:44,752 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:55:44,753 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:55:44,753 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ba35ada4783f:0, corePoolSize=5, maxPoolSize=5 2024-11-24T09:55:44,753 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ba35ada4783f:0, corePoolSize=10, maxPoolSize=10 2024-11-24T09:55:44,753 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:44,754 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ba35ada4783f:0, corePoolSize=2, maxPoolSize=2 2024-11-24T09:55:44,754 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:44,754 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(746): ClusterId : d4c77c4a-341a-48a8-9f7b-ddcebd83d205 2024-11-24T09:55:44,757 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T09:55:44,757 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732442174757 2024-11-24T09:55:44,759 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T09:55:44,759 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T09:55:44,760 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T09:55:44,760 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T09:55:44,764 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T09:55:44,764 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T09:55:44,765 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T09:55:44,765 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:44,765 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T09:55:44,765 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T09:55:44,766 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:44,770 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T09:55:44,772 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T09:55:44,772 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T09:55:44,775 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T09:55:44,775 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T09:55:44,777 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.large.0-1732442144776,5,FailOnTimeoutGroup] 2024-11-24T09:55:44,777 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.small.0-1732442144777,5,FailOnTimeoutGroup] 2024-11-24T09:55:44,777 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:44,778 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T09:55:44,779 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:44,779 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:44,788 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T09:55:44,789 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c 2024-11-24T09:55:44,797 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:55:44,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T09:55:44,801 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T09:55:44,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:44,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:55:44,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T09:55:44,805 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T09:55:44,806 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:44,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:55:44,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T09:55:44,816 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T09:55:44,816 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:44,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:55:44,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T09:55:44,820 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T09:55:44,820 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:44,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:55:44,821 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T09:55:44,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740 2024-11-24T09:55:44,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740 2024-11-24T09:55:44,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T09:55:44,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T09:55:44,824 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T09:55:44,825 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T09:55:44,831 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:55:44,831 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64179993, jitterRate=-0.04364357888698578}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T09:55:44,833 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T09:55:44,834 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T09:55:44,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732442144797Initializing all the Stores at 1732442144797Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732442144797Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732442144797Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732442144797Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732442144797Cleaning up temporary data from old regions at 1732442144823 (+26 ms)Region opened successfully at 1732442144836 (+13 ms) 2024-11-24T09:55:44,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T09:55:44,837 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T09:55:44,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T09:55:44,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T09:55:44,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T09:55:44,839 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T09:55:44,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732442144837Disabling compacts and flushes for region at 1732442144837Disabling writes for close at 1732442144838 (+1 ms)Writing region close event to WAL at 1732442144839 (+1 ms)Closed at 1732442144839 2024-11-24T09:55:44,843 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T09:55:44,843 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T09:55:44,849 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T09:55:44,850 DEBUG [RS:0;ba35ada4783f:42519 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f97d53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba35ada4783f/172.17.0.2:0 2024-11-24T09:55:44,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T09:55:44,861 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T09:55:44,863 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T09:55:44,866 DEBUG [RS:0;ba35ada4783f:42519 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ba35ada4783f:42519 2024-11-24T09:55:44,869 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T09:55:44,869 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T09:55:44,869 DEBUG [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T09:55:44,872 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba35ada4783f,36847,1732442142820 with port=42519, startcode=1732442143623 2024-11-24T09:55:44,881 DEBUG [RS:0;ba35ada4783f:42519 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T09:55:44,941 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47395, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T09:55:44,948 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36847 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba35ada4783f,42519,1732442143623 2024-11-24T09:55:44,951 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36847 {}] master.ServerManager(517): Registering regionserver=ba35ada4783f,42519,1732442143623 2024-11-24T09:55:44,965 DEBUG [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c 2024-11-24T09:55:44,965 DEBUG [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T09:55:44,965 DEBUG [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T09:55:45,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T09:55:45,007 DEBUG [RS:0;ba35ada4783f:42519 {}] zookeeper.ZKUtil(111): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba35ada4783f,42519,1732442143623 2024-11-24T09:55:45,007 WARN [RS:0;ba35ada4783f:42519 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T09:55:45,007 INFO [RS:0;ba35ada4783f:42519 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:55:45,008 DEBUG [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/WALs/ba35ada4783f,42519,1732442143623 2024-11-24T09:55:45,011 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba35ada4783f,42519,1732442143623] 2024-11-24T09:55:45,014 WARN [ba35ada4783f:36847 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T09:55:45,034 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T09:55:45,050 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T09:55:45,055 INFO [RS:0;ba35ada4783f:42519 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T09:55:45,055 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:45,056 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T09:55:45,062 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T09:55:45,064 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:45,064 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:45,064 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:45,064 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:45,065 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:45,065 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:45,065 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba35ada4783f:0, corePoolSize=2, maxPoolSize=2 2024-11-24T09:55:45,065 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:45,065 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:45,066 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:45,066 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:45,066 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:45,066 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba35ada4783f:0, corePoolSize=1, maxPoolSize=1 2024-11-24T09:55:45,066 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:55:45,067 DEBUG [RS:0;ba35ada4783f:42519 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba35ada4783f:0, corePoolSize=3, maxPoolSize=3 2024-11-24T09:55:45,068 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:45,068 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:45,068 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:45,068 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:45,068 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:45,069 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,42519,1732442143623-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T09:55:45,092 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T09:55:45,094 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,42519,1732442143623-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:45,094 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:45,094 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.Replication(171): ba35ada4783f,42519,1732442143623 started 2024-11-24T09:55:45,112 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:45,113 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(1482): Serving as ba35ada4783f,42519,1732442143623, RpcServer on ba35ada4783f/172.17.0.2:42519, sessionid=0x1016d8ec9fe0001 2024-11-24T09:55:45,114 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T09:55:45,114 DEBUG [RS:0;ba35ada4783f:42519 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba35ada4783f,42519,1732442143623 2024-11-24T09:55:45,115 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,42519,1732442143623' 2024-11-24T09:55:45,115 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T09:55:45,116 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T09:55:45,117 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T09:55:45,117 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T09:55:45,118 DEBUG [RS:0;ba35ada4783f:42519 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba35ada4783f,42519,1732442143623 2024-11-24T09:55:45,118 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba35ada4783f,42519,1732442143623' 2024-11-24T09:55:45,118 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T09:55:45,119 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T09:55:45,120 DEBUG [RS:0;ba35ada4783f:42519 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T09:55:45,120 INFO [RS:0;ba35ada4783f:42519 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T09:55:45,120 INFO [RS:0;ba35ada4783f:42519 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T09:55:45,228 INFO [RS:0;ba35ada4783f:42519 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T09:55:45,230 INFO [RS:0;ba35ada4783f:42519 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ba35ada4783f%2C42519%2C1732442143623, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/WALs/ba35ada4783f,42519,1732442143623, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/oldWALs, maxLogs=55 2024-11-24T09:55:45,253 INFO [RS:0;ba35ada4783f:42519 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/WALs/ba35ada4783f,42519,1732442143623/ba35ada4783f%2C42519%2C1732442143623.1732442145236 2024-11-24T09:55:45,254 DEBUG [RS:0;ba35ada4783f:42519 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T09:55:45,518 DEBUG [ba35ada4783f:36847 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T09:55:45,534 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba35ada4783f,42519,1732442143623 2024-11-24T09:55:45,538 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba35ada4783f,42519,1732442143623, state=OPENING 2024-11-24T09:55:45,598 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T09:55:45,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:45,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:45,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:55:45,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:55:45,608 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T09:55:45,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba35ada4783f,42519,1732442143623}] 2024-11-24T09:55:45,793 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:55:45,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42091, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:55:45,803 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T09:55:45,803 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T09:55:45,804 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T09:55:45,805 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ba35ada4783f%2C42519%2C1732442143623.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/WALs/ba35ada4783f,42519,1732442143623, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/oldWALs, maxLogs=55 2024-11-24T09:55:45,823 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/WALs/ba35ada4783f,42519,1732442143623/ba35ada4783f%2C42519%2C1732442143623.meta.1732442145807.meta 2024-11-24T09:55:45,824 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T09:55:45,824 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:55:45,826 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T09:55:45,828 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T09:55:45,832 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T09:55:45,836 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T09:55:45,836 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:55:45,837 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T09:55:45,837 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T09:55:45,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T09:55:45,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T09:55:45,839 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:45,839 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:55:45,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T09:55:45,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T09:55:45,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:45,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:55:45,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T09:55:45,841 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T09:55:45,841 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:45,842 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:55:45,842 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T09:55:45,842 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T09:55:45,842 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:45,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T09:55:45,843 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T09:55:45,844 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740 2024-11-24T09:55:45,845 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740 2024-11-24T09:55:45,845 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T09:55:45,845 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T09:55:45,845 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T09:55:45,847 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T09:55:45,847 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74894760, jitterRate=0.11601889133453369}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T09:55:45,847 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T09:55:45,849 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732442145837Writing region info on filesystem at 1732442145838 (+1 ms)Initializing all the Stores at 1732442145838Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732442145838Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732442145838Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732442145838Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732442145838Cleaning up temporary data from old regions at 1732442145845 (+7 ms)Running coprocessor post-open hooks at 1732442145847 (+2 ms)Region opened successfully at 1732442145849 (+2 ms) 2024-11-24T09:55:45,853 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732442145784 2024-11-24T09:55:45,862 DEBUG [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T09:55:45,863 INFO [RS_OPEN_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T09:55:45,863 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ba35ada4783f,42519,1732442143623 2024-11-24T09:55:45,864 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba35ada4783f,42519,1732442143623, state=OPEN 2024-11-24T09:55:45,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:55:45,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T09:55:45,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:55:45,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T09:55:45,920 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba35ada4783f,42519,1732442143623 2024-11-24T09:55:45,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T09:55:45,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba35ada4783f,42519,1732442143623 in 310 msec 2024-11-24T09:55:45,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T09:55:45,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0740 sec 2024-11-24T09:55:45,929 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T09:55:45,929 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T09:55:45,948 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:55:45,949 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42519,1732442143623, seqNum=-1] 2024-11-24T09:55:45,964 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T09:55:45,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56439, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T09:55:45,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2990 sec 2024-11-24T09:55:45,983 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732442145983, completionTime=-1 2024-11-24T09:55:45,985 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T09:55:45,986 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T09:55:46,009 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T09:55:46,009 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732442206009 2024-11-24T09:55:46,009 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732442266009 2024-11-24T09:55:46,009 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-24T09:55:46,011 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,36847,1732442142820-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:46,012 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,36847,1732442142820-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:46,012 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,36847,1732442142820-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:46,013 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ba35ada4783f:36847, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:46,013 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:46,014 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:46,019 DEBUG [master/ba35ada4783f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T09:55:46,038 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.181sec 2024-11-24T09:55:46,039 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T09:55:46,040 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T09:55:46,040 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T09:55:46,041 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T09:55:46,041 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T09:55:46,041 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,36847,1732442142820-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T09:55:46,042 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,36847,1732442142820-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T09:55:46,045 DEBUG [master/ba35ada4783f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T09:55:46,046 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T09:55:46,046 INFO [master/ba35ada4783f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba35ada4783f,36847,1732442142820-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T09:55:46,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567b1492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T09:55:46,063 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T09:55:46,064 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T09:55:46,067 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ba35ada4783f,36847,-1 for getting cluster id 2024-11-24T09:55:46,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T09:55:46,077 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd4c77c4a-341a-48a8-9f7b-ddcebd83d205' 2024-11-24T09:55:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T09:55:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d4c77c4a-341a-48a8-9f7b-ddcebd83d205" 2024-11-24T09:55:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1bd0f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T09:55:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ba35ada4783f,36847,-1] 2024-11-24T09:55:46,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T09:55:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:55:46,091 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T09:55:46,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3d5ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T09:55:46,095 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T09:55:46,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba35ada4783f,42519,1732442143623, seqNum=-1] 2024-11-24T09:55:46,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T09:55:46,121 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T09:55:46,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ba35ada4783f,36847,1732442142820 2024-11-24T09:55:46,144 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T09:55:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ba35ada4783f,36847,1732442142820 2024-11-24T09:55:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43f05e06 2024-11-24T09:55:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T09:55:46,153 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T09:55:46,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36847 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T09:55:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36847 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-24T09:55:46,166 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T09:55:46,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36847 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-24T09:55:46,169 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:46,170 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T09:55:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36847 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:55:46,182 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f897df87f9def86dbdc75a1cc0ae8cd1, NAME => 'table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c 2024-11-24T09:55:46,189 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:55:46,189 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing f897df87f9def86dbdc75a1cc0ae8cd1, disabling compactions & flushes 2024-11-24T09:55:46,189 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:46,189 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:46,189 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. after waiting 0 ms 2024-11-24T09:55:46,189 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:46,189 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:46,189 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for f897df87f9def86dbdc75a1cc0ae8cd1: Waiting for close lock at 1732442146189Disabling compacts and flushes for region at 1732442146189Disabling writes for close at 1732442146189Writing region close event to WAL at 1732442146189Closed at 1732442146189 2024-11-24T09:55:46,191 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T09:55:46,195 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1732442146191"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732442146191"}]},"ts":"1732442146191"} 2024-11-24T09:55:46,200 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T09:55:46,201 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T09:55:46,203 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732442146201"}]},"ts":"1732442146201"} 2024-11-24T09:55:46,207 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-24T09:55:46,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=f897df87f9def86dbdc75a1cc0ae8cd1, ASSIGN}] 2024-11-24T09:55:46,211 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=f897df87f9def86dbdc75a1cc0ae8cd1, ASSIGN 2024-11-24T09:55:46,212 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=f897df87f9def86dbdc75a1cc0ae8cd1, ASSIGN; state=OFFLINE, location=ba35ada4783f,42519,1732442143623; forceNewPlan=false, retain=false 2024-11-24T09:55:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36847 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:55:46,364 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f897df87f9def86dbdc75a1cc0ae8cd1, regionState=OPENING, regionLocation=ba35ada4783f,42519,1732442143623 2024-11-24T09:55:46,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=f897df87f9def86dbdc75a1cc0ae8cd1, ASSIGN because future has completed 2024-11-24T09:55:46,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f897df87f9def86dbdc75a1cc0ae8cd1, server=ba35ada4783f,42519,1732442143623}] 2024-11-24T09:55:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36847 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:55:46,531 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:46,531 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f897df87f9def86dbdc75a1cc0ae8cd1, NAME => 'table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T09:55:46,532 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:46,533 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T09:55:46,533 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:46,533 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:46,535 INFO [StoreOpener-f897df87f9def86dbdc75a1cc0ae8cd1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:46,539 INFO [StoreOpener-f897df87f9def86dbdc75a1cc0ae8cd1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f897df87f9def86dbdc75a1cc0ae8cd1 columnFamilyName family 2024-11-24T09:55:46,539 DEBUG [StoreOpener-f897df87f9def86dbdc75a1cc0ae8cd1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T09:55:46,540 INFO [StoreOpener-f897df87f9def86dbdc75a1cc0ae8cd1-1 {}] regionserver.HStore(327): Store=f897df87f9def86dbdc75a1cc0ae8cd1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T09:55:46,540 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:46,541 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:46,541 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:46,541 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:46,541 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:46,542 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:46,547 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T09:55:46,547 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f897df87f9def86dbdc75a1cc0ae8cd1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63525463, jitterRate=-0.053396835923194885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T09:55:46,548 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:46,548 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f897df87f9def86dbdc75a1cc0ae8cd1: Running coprocessor pre-open hook at 1732442146534Writing region info on filesystem at 1732442146534Initializing all the Stores at 1732442146534Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732442146534Cleaning up temporary data from old regions at 1732442146541 (+7 ms)Running coprocessor post-open hooks at 1732442146548 (+7 ms)Region opened successfully at 1732442146548 2024-11-24T09:55:46,549 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1., pid=6, masterSystemTime=1732442146523 2024-11-24T09:55:46,552 DEBUG [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:46,553 INFO [RS_OPEN_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:46,553 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f897df87f9def86dbdc75a1cc0ae8cd1, regionState=OPEN, openSeqNum=2, regionLocation=ba35ada4783f,42519,1732442143623 2024-11-24T09:55:46,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f897df87f9def86dbdc75a1cc0ae8cd1, server=ba35ada4783f,42519,1732442143623 because future has completed 2024-11-24T09:55:46,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T09:55:46,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f897df87f9def86dbdc75a1cc0ae8cd1, server=ba35ada4783f,42519,1732442143623 in 189 msec 2024-11-24T09:55:46,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T09:55:46,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=f897df87f9def86dbdc75a1cc0ae8cd1, ASSIGN in 353 msec 2024-11-24T09:55:46,565 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T09:55:46,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732442146565"}]},"ts":"1732442146565"} 2024-11-24T09:55:46,568 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-24T09:55:46,569 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T09:55:46,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 408 msec 2024-11-24T09:55:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36847 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T09:55:46,802 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-24T09:55:46,802 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-24T09:55:46,806 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T09:55:46,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-24T09:55:46,816 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T09:55:46,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-24T09:55:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1., hostname=ba35ada4783f,42519,1732442143623, seqNum=2] 2024-11-24T09:55:46,843 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f897df87f9def86dbdc75a1cc0ae8cd1 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T09:55:46,871 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dd8996ea6b244e069e8f38abb6e7b276 2024-11-24T09:55:46,906 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/.tmp/family/dd8996ea6b244e069e8f38abb6e7b276 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T09:55:46,911 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/.tmp/family/dd8996ea6b244e069e8f38abb6e7b276 2024-11-24T09:55:46,941 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/.tmp/family/dd8996ea6b244e069e8f38abb6e7b276 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/family/dd8996ea6b244e069e8f38abb6e7b276 2024-11-24T09:55:46,943 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/family/dd8996ea6b244e069e8f38abb6e7b276, delay=960 ms 2024-11-24T09:55:46,944 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/family/dd8996ea6b244e069e8f38abb6e7b276, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T09:55:46,950 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for f897df87f9def86dbdc75a1cc0ae8cd1 in 107ms, sequenceid=6, compaction requested=false 2024-11-24T09:55:46,951 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-24T09:55:46,953 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f897df87f9def86dbdc75a1cc0ae8cd1: 2024-11-24T09:55:47,905 DEBUG [hfile-prefetch-1732442146943 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key dd8996ea6b244e069e8f38abb6e7b276_0. 2024-11-24T09:55:47,910 DEBUG [hfile-prefetch-1732442146943 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key dd8996ea6b244e069e8f38abb6e7b276_117. 2024-11-24T09:55:47,911 DEBUG [hfile-prefetch-1732442146943 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/family/dd8996ea6b244e069e8f38abb6e7b276, with total blocks 2, and data blocks 1 2024-11-24T09:55:47,911 DEBUG [hfile-prefetch-1732442146943 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T09:55:47,911 DEBUG [hfile-prefetch-1732442146943 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/family/dd8996ea6b244e069e8f38abb6e7b276, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T09:55:47,912 DEBUG [hfile-prefetch-1732442146943 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/family/dd8996ea6b244e069e8f38abb6e7b276. Will sleep 100ms and try the verification again. 2024-11-24T09:55:48,012 DEBUG [hfile-prefetch-1732442146943 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/family/dd8996ea6b244e069e8f38abb6e7b276, with total blocks 2, and data blocks 1 2024-11-24T09:55:48,012 DEBUG [hfile-prefetch-1732442146943 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-24T09:55:48,012 DEBUG [hfile-prefetch-1732442146943 {}] bucket.BucketCache(2293): found block for file dd8996ea6b244e069e8f38abb6e7b276 in the backing map. Acquiring read lock for offset 0 2024-11-24T09:55:48,012 DEBUG [hfile-prefetch-1732442146943 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/family/dd8996ea6b244e069e8f38abb6e7b276 has now been fully cached. 2024-11-24T09:55:48,013 DEBUG [hfile-prefetch-1732442146943 {}] hfile.PrefetchExecutor(121): Prefetch completed for dd8996ea6b244e069e8f38abb6e7b276 2024-11-24T09:55:48,723 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T09:55:48,723 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T09:55:48,724 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T09:55:48,724 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-24T09:55:48,747 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-24T09:55:48,747 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732442148723 2024-11-24T09:55:48,748 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T09:55:48,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T09:55:48,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T09:55:48,961 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;ba35ada4783f:42519,5,FailOnTimeoutGroup] 2024-11-24T09:55:48,962 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba35ada4783f,42519,1732442143623' ***** 2024-11-24T09:55:48,962 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-24T09:55:48,962 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-24T09:55:48,962 INFO [RS:0;ba35ada4783f:42519 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T09:55:48,962 INFO [RS:0;ba35ada4783f:42519 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/bucket.persistence 2024-11-24T09:55:48,963 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T09:55:48,963 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T09:55:48,963 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T09:55:48,963 INFO [RS:0;ba35ada4783f:42519 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T09:55:48,963 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T09:55:48,973 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T09:55:48,973 DEBUG [RS:0;ba35ada4783f:42519 {}] bucket.BucketCache(1487): Thread RS:0;ba35ada4783f:42519 started persisting bucket cache to file 2024-11-24T09:55:48,973 DEBUG [RS:0;ba35ada4783f:42519 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T09:55:48,973 DEBUG [RS:0;ba35ada4783f:42519 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-24T09:55:48,973 INFO [regionserver/ba35ada4783f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:55:48,976 DEBUG [RS:0;ba35ada4783f:42519 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-24T09:55:48,976 DEBUG [RS:0;ba35ada4783f:42519 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732442148973 2024-11-24T09:55:48,976 DEBUG [RS:0;ba35ada4783f:42519 {}] bucket.BucketCache(1509): Thread RS:0;ba35ada4783f:42519 finished persisting bucket cache to file, renaming 2024-11-24T09:55:49,010 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T09:55:49,010 INFO [RS:0;ba35ada4783f:42519 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T09:55:49,011 INFO [RS:0;ba35ada4783f:42519 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T09:55:49,011 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(3091): Received CLOSE for f897df87f9def86dbdc75a1cc0ae8cd1 2024-11-24T09:55:49,012 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(959): stopping server ba35ada4783f,42519,1732442143623 2024-11-24T09:55:49,012 INFO [RS:0;ba35ada4783f:42519 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T09:55:49,013 INFO [RS:0;ba35ada4783f:42519 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ba35ada4783f:42519. 2024-11-24T09:55:49,013 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T09:55:49,013 DEBUG [RS:0;ba35ada4783f:42519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:55:49,013 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f897df87f9def86dbdc75a1cc0ae8cd1, disabling compactions & flushes 2024-11-24T09:55:49,014 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:49,014 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:49,014 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. after waiting 0 ms 2024-11-24T09:55:49,014 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:49,022 DEBUG [RS:0;ba35ada4783f:42519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:55:49,022 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/default/table1/f897df87f9def86dbdc75a1cc0ae8cd1/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T09:55:49,023 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T09:55:49,023 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T09:55:49,023 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T09:55:49,023 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T09:55:49,023 INFO [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:49,023 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f897df87f9def86dbdc75a1cc0ae8cd1: Waiting for close lock at 1732442149012Running coprocessor pre-close hooks at 1732442149013 (+1 ms)Disabling compacts and flushes for region at 1732442149013Disabling writes for close at 1732442149014 (+1 ms)Writing region close event to WAL at 1732442149017 (+3 ms)Running coprocessor post-close hooks at 1732442149022 (+5 ms)Closed at 1732442149023 (+1 ms) 2024-11-24T09:55:49,024 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T09:55:49,024 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T09:55:49,024 DEBUG [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T09:55:49,024 DEBUG [RS_CLOSE_REGION-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1. 2024-11-24T09:55:49,024 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T09:55:49,024 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T09:55:49,024 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T09:55:49,024 DEBUG [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T09:55:49,024 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T09:55:49,024 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T09:55:49,024 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-24T09:55:49,032 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d2c9b7d357ee4819912304be7dcdf219 2024-11-24T09:55:49,049 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/.tmp/info/d2c9b7d357ee4819912304be7dcdf219 is 133, key is table1,,1732442146154.f897df87f9def86dbdc75a1cc0ae8cd1./info:regioninfo/1732442146553/Put/seqid=0 2024-11-24T09:55:49,049 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/.tmp/info/d2c9b7d357ee4819912304be7dcdf219 2024-11-24T09:55:49,059 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f5b7913bccf142729e69912b6c069355 2024-11-24T09:55:49,068 INFO [regionserver/ba35ada4783f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T09:55:49,068 INFO [regionserver/ba35ada4783f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T09:55:49,072 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/.tmp/ns/f5b7913bccf142729e69912b6c069355 is 43, key is default/ns:d/1732442145970/Put/seqid=0 2024-11-24T09:55:49,072 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/.tmp/ns/f5b7913bccf142729e69912b6c069355 2024-11-24T09:55:49,080 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c54f835c7dc64f0b94785f980f764271 2024-11-24T09:55:49,092 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/.tmp/table/c54f835c7dc64f0b94785f980f764271 is 42, key is table1/table:state/1732442146565/Put/seqid=0 2024-11-24T09:55:49,093 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/.tmp/table/c54f835c7dc64f0b94785f980f764271 2024-11-24T09:55:49,095 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/.tmp/info/d2c9b7d357ee4819912304be7dcdf219 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/info/d2c9b7d357ee4819912304be7dcdf219 2024-11-24T09:55:49,096 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/info/d2c9b7d357ee4819912304be7dcdf219, delay=910 ms 2024-11-24T09:55:49,098 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/info/d2c9b7d357ee4819912304be7dcdf219, entries=10, sequenceid=11, filesize=6.3 K 2024-11-24T09:55:49,098 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/.tmp/ns/f5b7913bccf142729e69912b6c069355 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/ns/f5b7913bccf142729e69912b6c069355 2024-11-24T09:55:49,099 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/ns/f5b7913bccf142729e69912b6c069355, delay=977 ms 2024-11-24T09:55:49,101 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/ns/f5b7913bccf142729e69912b6c069355, entries=2, sequenceid=11, filesize=5.0 K 2024-11-24T09:55:49,101 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/.tmp/table/c54f835c7dc64f0b94785f980f764271 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/table/c54f835c7dc64f0b94785f980f764271 2024-11-24T09:55:49,102 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/table/c54f835c7dc64f0b94785f980f764271, delay=952 ms 2024-11-24T09:55:49,104 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/table/c54f835c7dc64f0b94785f980f764271, entries=2, sequenceid=11, filesize=5.1 K 2024-11-24T09:55:49,104 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=11, compaction requested=false 2024-11-24T09:55:49,104 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T09:55:49,106 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/info/d2c9b7d357ee4819912304be7dcdf219 2024-11-24T09:55:49,107 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/ns/f5b7913bccf142729e69912b6c069355 2024-11-24T09:55:49,107 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/table/c54f835c7dc64f0b94785f980f764271 2024-11-24T09:55:49,115 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-24T09:55:49,115 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T09:55:49,115 INFO [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T09:55:49,116 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732442149024Running coprocessor pre-close hooks at 1732442149024Disabling compacts and flushes for region at 1732442149024Disabling writes for close at 1732442149024Obtaining lock to block concurrent updates at 1732442149024Preparing flush snapshotting stores in 1588230740 at 1732442149024Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1732442149025 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732442149025Flushing 1588230740/info: creating writer at 1732442149026 (+1 ms)Flushing 1588230740/info: appending metadata at 1732442149048 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732442149048Flushing 1588230740/ns: creating writer at 1732442149052 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732442149071 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732442149071Flushing 1588230740/table: creating writer at 1732442149075 (+4 ms)Flushing 1588230740/table: appending metadata at 1732442149092 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732442149092Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7793c0e3: reopening flushed file at 1732442149095 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c16d80a: reopening flushed file at 1732442149098 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40131d05: reopening flushed file at 1732442149101 (+3 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=11, compaction requested=false at 1732442149104 (+3 ms)Writing region close event to WAL at 1732442149108 (+4 ms)Running coprocessor post-close hooks at 1732442149115 (+7 ms)Closed at 1732442149115 2024-11-24T09:55:49,116 DEBUG [RS_CLOSE_META-regionserver/ba35ada4783f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T09:55:49,225 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(976): stopping server ba35ada4783f,42519,1732442143623; all regions closed. 2024-11-24T09:55:49,232 DEBUG [RS:0;ba35ada4783f:42519 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/oldWALs 2024-11-24T09:55:49,233 INFO [RS:0;ba35ada4783f:42519 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ba35ada4783f%2C42519%2C1732442143623.meta:.meta(num 1732442145807) 2024-11-24T09:55:49,237 DEBUG [RS:0;ba35ada4783f:42519 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/oldWALs 2024-11-24T09:55:49,237 INFO [RS:0;ba35ada4783f:42519 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ba35ada4783f%2C42519%2C1732442143623:(num 1732442145236) 2024-11-24T09:55:49,237 DEBUG [RS:0;ba35ada4783f:42519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:55:49,237 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T09:55:49,238 INFO [RS:0;ba35ada4783f:42519 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T09:55:49,238 INFO [RS:0;ba35ada4783f:42519 {}] hbase.ChoreService(370): Chore service for: regionserver/ba35ada4783f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T09:55:49,238 INFO [RS:0;ba35ada4783f:42519 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T09:55:49,239 INFO [regionserver/ba35ada4783f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T09:55:49,239 INFO [RS:0;ba35ada4783f:42519 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42519 2024-11-24T09:55:49,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba35ada4783f,42519,1732442143623 2024-11-24T09:55:49,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T09:55:49,272 INFO [RS:0;ba35ada4783f:42519 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T09:55:49,281 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba35ada4783f,42519,1732442143623] 2024-11-24T09:55:49,288 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba35ada4783f,42519,1732442143623 already deleted, retry=false 2024-11-24T09:55:49,289 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ba35ada4783f,42519,1732442143623 on ba35ada4783f,36847,1732442142820 2024-11-24T09:55:49,295 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ba35ada4783f,42519,1732442143623, splitWal=true, meta=true 2024-11-24T09:55:49,299 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for ba35ada4783f,42519,1732442143623 (carryingMeta=true) ba35ada4783f,42519,1732442143623/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3eb64a11[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T09:55:49,300 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ba35ada4783f,42519,1732442143623, splitWal=true, meta=true 2024-11-24T09:55:49,301 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ba35ada4783f,42519,1732442143623, splitWal=true, meta=true, isMeta: true 2024-11-24T09:55:49,302 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/WALs/ba35ada4783f,42519,1732442143623-splitting 2024-11-24T09:55:49,302 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/WALs/ba35ada4783f,42519,1732442143623-splitting dir is empty, no logs to split. 2024-11-24T09:55:49,302 INFO [PEWorker-2 {}] master.SplitWALManager(105): ba35ada4783f,42519,1732442143623 WAL count=0, meta=true 2024-11-24T09:55:49,304 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/WALs/ba35ada4783f,42519,1732442143623-splitting dir is empty, no logs to split. 2024-11-24T09:55:49,304 INFO [PEWorker-2 {}] master.SplitWALManager(105): ba35ada4783f,42519,1732442143623 WAL count=0, meta=true 2024-11-24T09:55:49,305 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if ba35ada4783f,42519,1732442143623 WAL splitting is done? wals=0, meta=true 2024-11-24T09:55:49,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T09:55:49,307 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T09:55:49,308 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T09:55:49,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:55:49,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42519-0x1016d8ec9fe0001, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:55:49,381 INFO [RS:0;ba35ada4783f:42519 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T09:55:49,381 INFO [RS:0;ba35ada4783f:42519 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba35ada4783f,42519,1732442143623; zookeeper connection closed. 2024-11-24T09:55:49,382 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2344066d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2344066d 2024-11-24T09:55:49,458 WARN [ba35ada4783f:36847 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T09:55:49,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T09:55:49,963 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T09:55:49,963 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:55:49,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:55:49,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:55:49,965 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T09:55:49,965 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T09:55:49,965 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=365092990, stopped=false 2024-11-24T09:55:49,966 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ba35ada4783f,36847,1732442142820 2024-11-24T09:55:49,966 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'ba35ada4783f,36847,1732442142820' ***** 2024-11-24T09:55:49,966 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-24T09:55:49,967 INFO [M:0;ba35ada4783f:36847 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T09:55:49,967 INFO [M:0;ba35ada4783f:36847 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;ba35ada4783f:36847. 2024-11-24T09:55:49,967 DEBUG [M:0;ba35ada4783f:36847 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T09:55:49,967 DEBUG [M:0;ba35ada4783f:36847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T09:55:49,967 INFO [M:0;ba35ada4783f:36847 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T09:55:49,967 DEBUG [M:0;ba35ada4783f:36847 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T09:55:49,968 DEBUG [M:0;ba35ada4783f:36847 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T09:55:49,968 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T09:55:49,968 DEBUG [master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.large.0-1732442144776 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.large.0-1732442144776,5,FailOnTimeoutGroup] 2024-11-24T09:55:49,968 DEBUG [master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.small.0-1732442144777 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba35ada4783f:0:becomeActiveMaster-HFileCleaner.small.0-1732442144777,5,FailOnTimeoutGroup] 2024-11-24T09:55:49,968 INFO [M:0;ba35ada4783f:36847 {}] hbase.ChoreService(370): Chore service for: master/ba35ada4783f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T09:55:49,969 INFO [M:0;ba35ada4783f:36847 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T09:55:49,969 DEBUG [M:0;ba35ada4783f:36847 {}] master.HMaster(1795): Stopping service threads 2024-11-24T09:55:49,969 INFO [M:0;ba35ada4783f:36847 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T09:55:49,969 INFO [M:0;ba35ada4783f:36847 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T09:55:49,971 INFO [M:0;ba35ada4783f:36847 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T09:55:49,971 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T09:55:50,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T09:55:50,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T09:55:50,085 DEBUG [M:0;ba35ada4783f:36847 {}] zookeeper.ZKUtil(347): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T09:55:50,085 WARN [M:0;ba35ada4783f:36847 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T09:55:50,086 INFO [M:0;ba35ada4783f:36847 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/.lastflushedseqids 2024-11-24T09:55:50,099 INFO [M:0;ba35ada4783f:36847 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T09:55:50,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T09:55:50,150 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T09:55:50,151 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T09:55:50,209 DEBUG [ba35ada4783f:36847 {}] assignment.AssignmentManager(2455): Stopped! Dropping assign of 1 queued regions. 2024-11-24T09:55:50,209 INFO [M:0;ba35ada4783f:36847 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T09:55:50,210 DEBUG [M:0;ba35ada4783f:36847 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T09:55:50,210 INFO [M:0;ba35ada4783f:36847 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:55:50,211 DEBUG [M:0;ba35ada4783f:36847 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:55:50,211 DEBUG [M:0;ba35ada4783f:36847 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T09:55:50,211 DEBUG [M:0;ba35ada4783f:36847 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:55:50,212 INFO [M:0;ba35ada4783f:36847 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.17 KB heapSize=30.80 KB 2024-11-24T09:55:50,226 DEBUG [M:0;ba35ada4783f:36847 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 84f9fc272fe04955b1a3302bceac4600 2024-11-24T09:55:50,244 DEBUG [M:0;ba35ada4783f:36847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/84f9fc272fe04955b1a3302bceac4600 is 82, key is hbase:meta,,1/info:regioninfo/1732442145863/Put/seqid=0 2024-11-24T09:55:50,245 INFO [M:0;ba35ada4783f:36847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/84f9fc272fe04955b1a3302bceac4600 2024-11-24T09:55:50,264 DEBUG [M:0;ba35ada4783f:36847 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 472417e015654f68906fa2dc81e20ce4 2024-11-24T09:55:50,282 DEBUG [M:0;ba35ada4783f:36847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/472417e015654f68906fa2dc81e20ce4 is 706, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732442146571/Put/seqid=0 2024-11-24T09:55:50,283 INFO [M:0;ba35ada4783f:36847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.56 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/472417e015654f68906fa2dc81e20ce4 2024-11-24T09:55:50,291 DEBUG [M:0;ba35ada4783f:36847 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 04ad43afd3594ad5bf227dd8c772fe94 2024-11-24T09:55:50,303 DEBUG [M:0;ba35ada4783f:36847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/04ad43afd3594ad5bf227dd8c772fe94 is 58, key is ba35ada4783f,42519,1732442143623/rs:/1732442149300/DeleteFamily/seqid=0 2024-11-24T09:55:50,303 INFO [M:0;ba35ada4783f:36847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/04ad43afd3594ad5bf227dd8c772fe94 2024-11-24T09:55:50,305 INFO [M:0;ba35ada4783f:36847 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 04ad43afd3594ad5bf227dd8c772fe94 2024-11-24T09:55:50,305 DEBUG [M:0;ba35ada4783f:36847 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/84f9fc272fe04955b1a3302bceac4600 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/84f9fc272fe04955b1a3302bceac4600 2024-11-24T09:55:50,307 INFO [M:0;ba35ada4783f:36847 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/84f9fc272fe04955b1a3302bceac4600, entries=8, sequenceid=64, filesize=5.5 K 2024-11-24T09:55:50,307 DEBUG [M:0;ba35ada4783f:36847 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/472417e015654f68906fa2dc81e20ce4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/472417e015654f68906fa2dc81e20ce4 2024-11-24T09:55:50,309 INFO [M:0;ba35ada4783f:36847 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/472417e015654f68906fa2dc81e20ce4, entries=8, sequenceid=64, filesize=6.5 K 2024-11-24T09:55:50,309 DEBUG [M:0;ba35ada4783f:36847 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/04ad43afd3594ad5bf227dd8c772fe94 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/04ad43afd3594ad5bf227dd8c772fe94 2024-11-24T09:55:50,310 INFO [M:0;ba35ada4783f:36847 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 04ad43afd3594ad5bf227dd8c772fe94 2024-11-24T09:55:50,311 INFO [M:0;ba35ada4783f:36847 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5af9a0e8-6917-9dc6-3838-dfa1f03d1f2a/45353323-8161-2a39-689c-73337dec205c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/04ad43afd3594ad5bf227dd8c772fe94, entries=1, sequenceid=64, filesize=5.2 K 2024-11-24T09:55:50,311 INFO [M:0;ba35ada4783f:36847 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=64, compaction requested=false 2024-11-24T09:55:50,313 INFO [M:0;ba35ada4783f:36847 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T09:55:50,314 DEBUG [M:0;ba35ada4783f:36847 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732442150210Disabling compacts and flushes for region at 1732442150210Disabling writes for close at 1732442150211 (+1 ms)Obtaining lock to block concurrent updates at 1732442150213 (+2 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732442150213Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24747, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1732442150214 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732442150215 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732442150215Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732442150243 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732442150244 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732442150247 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732442150282 (+35 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732442150282Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732442150286 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732442150302 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732442150302Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8e13b23: reopening flushed file at 1732442150305 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29e2ac0d: reopening flushed file at 1732442150307 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@282af41e: reopening flushed file at 1732442150309 (+2 ms)Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=64, compaction requested=false at 1732442150311 (+2 ms)Writing region close event to WAL at 1732442150313 (+2 ms)Closed at 1732442150313 2024-11-24T09:55:50,315 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T09:55:50,315 INFO [M:0;ba35ada4783f:36847 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T09:55:50,316 INFO [M:0;ba35ada4783f:36847 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36847 2024-11-24T09:55:50,316 INFO [M:0;ba35ada4783f:36847 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T09:55:50,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:55:50,422 INFO [M:0;ba35ada4783f:36847 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T09:55:50,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36847-0x1016d8ec9fe0000, quorum=127.0.0.1:51787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T09:55:50,426 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T09:55:50,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T09:55:50,452 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T09:55:50,459 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=45 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ba35ada4783f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732442149100 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732442149102 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732442146943 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732442149097 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ba35ada4783f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ba35ada4783f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=253 (was 273), ProcessCount=11 (was 11), AvailableMemoryMB=8289 (was 8589)