2024-11-13 18:19:54,748 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 18:19:54,786 main DEBUG Took 0.033652 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 18:19:54,786 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 18:19:54,787 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 18:19:54,788 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 18:19:54,789 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,801 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 18:19:54,833 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,836 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,836 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,837 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,837 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,838 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,838 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,838 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,839 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,840 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,840 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,841 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,841 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,842 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,842 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,843 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,843 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,843 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,844 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,844 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,845 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,845 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,846 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:19:54,846 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,851 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 18:19:54,854 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:19:54,855 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 18:19:54,858 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 18:19:54,858 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 18:19:54,860 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 18:19:54,861 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 18:19:54,874 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 18:19:54,877 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 18:19:54,880 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 18:19:54,880 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 18:19:54,881 main DEBUG createAppenders(={Console}) 2024-11-13 18:19:54,882 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-13 18:19:54,882 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 18:19:54,882 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-13 18:19:54,883 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 18:19:54,884 main DEBUG OutputStream closed 2024-11-13 18:19:54,885 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 18:19:54,885 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 18:19:54,886 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-13 18:19:54,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 18:19:54,991 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 18:19:54,992 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 18:19:54,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 18:19:54,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 18:19:54,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 18:19:54,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 18:19:54,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 18:19:54,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 18:19:54,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 18:19:54,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 18:19:54,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 18:19:54,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 18:19:54,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 18:19:54,999 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 18:19:54,999 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 18:19:55,000 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 18:19:55,001 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 18:19:55,004 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 18:19:55,004 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-13 18:19:55,005 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 18:19:55,006 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-13T18:19:55,025 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-13 18:19:55,029 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 18:19:55,029 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T18:19:55,450 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018 2024-11-13T18:19:55,540 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=649, ProcessCount=12, AvailableMemoryMB=4372 2024-11-13T18:19:55,706 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T18:19:55,851 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:19:55,888 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/cluster_4646466e-8791-fdbe-61d4-6bab77133992, deleteOnExit=true 2024-11-13T18:19:56,029 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/cluster_4646466e-8791-fdbe-61d4-6bab77133992/zookeeper_0, clientPort=54759, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/cluster_4646466e-8791-fdbe-61d4-6bab77133992/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/cluster_4646466e-8791-fdbe-61d4-6bab77133992/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T18:19:56,046 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54759 2024-11-13T18:19:56,047 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:19:56,170 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15 with version=8 2024-11-13T18:19:56,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/hbase-staging 2024-11-13T18:19:56,301 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T18:19:56,680 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:19:56,697 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:19:56,699 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:19:56,709 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:19:56,710 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:19:56,710 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:19:56,875 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T18:19:56,979 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:19:57,025 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T18:19:57,035 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T18:19:57,040 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:19:57,086 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 20567 (auto-detected) 2024-11-13T18:19:57,087 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T18:19:57,116 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38923 2024-11-13T18:19:57,139 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38923 connecting to ZooKeeper ensemble=127.0.0.1:54759 2024-11-13T18:19:57,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:389230x0, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:19:57,182 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38923-0x1003a5ef7540000 connected 2024-11-13T18:19:57,216 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:19:57,218 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:19:57,228 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:19:57,233 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15, hbase.cluster.distributed=false 2024-11-13T18:19:57,265 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:19:57,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38923 2024-11-13T18:19:57,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38923 2024-11-13T18:19:57,286 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38923 2024-11-13T18:19:57,288 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38923 2024-11-13T18:19:57,289 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38923 2024-11-13T18:19:57,423 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:19:57,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:19:57,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:19:57,427 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:19:57,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:19:57,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:19:57,431 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:19:57,434 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:19:57,435 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40065 2024-11-13T18:19:57,437 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40065 connecting to ZooKeeper ensemble=127.0.0.1:54759 2024-11-13T18:19:57,437 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:19:57,439 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:19:57,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400650x0, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:19:57,450 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:400650x0, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:19:57,452 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40065-0x1003a5ef7540001 connected 2024-11-13T18:19:57,457 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:19:57,474 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/bucket.cache 2024-11-13T18:19:57,482 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T18:19:57,484 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T18:19:57,485 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T18:19:57,492 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T18:19:57,492 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T18:19:57,501 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T18:19:57,510 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T18:19:57,511 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/bucket.persistence is complete. 2024-11-13T18:19:57,511 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:19:57,515 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:19:57,530 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:19:57,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40065 2024-11-13T18:19:57,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40065 2024-11-13T18:19:57,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40065 2024-11-13T18:19:57,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40065 2024-11-13T18:19:57,541 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40065 2024-11-13T18:19:57,571 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:19:57,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:19:57,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:19:57,572 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:19:57,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:19:57,573 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:19:57,573 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:19:57,573 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:19:57,576 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40625 2024-11-13T18:19:57,579 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40625 connecting to ZooKeeper ensemble=127.0.0.1:54759 2024-11-13T18:19:57,579 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:19:57,582 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:19:57,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406250x0, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:19:57,589 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:406250x0, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:19:57,590 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:19:57,595 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40625-0x1003a5ef7540002 connected 2024-11-13T18:19:57,610 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/bucket.cache 2024-11-13T18:19:57,610 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T18:19:57,611 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T18:19:57,611 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T18:19:57,616 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T18:19:57,618 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:19:57,620 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:19:57,622 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T18:19:57,622 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T18:19:57,623 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T18:19:57,624 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/bucket.persistence is complete. 2024-11-13T18:19:57,626 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:19:57,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40625 2024-11-13T18:19:57,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40625 2024-11-13T18:19:57,655 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40625 2024-11-13T18:19:57,663 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40625 2024-11-13T18:19:57,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40625 2024-11-13T18:19:57,700 DEBUG [M:0;fda96cf025dc:38923 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fda96cf025dc:38923 2024-11-13T18:19:57,702 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fda96cf025dc,38923,1731521996390 2024-11-13T18:19:57,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:19:57,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:19:57,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:19:57,722 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fda96cf025dc,38923,1731521996390 2024-11-13T18:19:57,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:19:57,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:57,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:57,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:19:57,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:57,766 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T18:19:57,768 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fda96cf025dc,38923,1731521996390 from backup master directory 2024-11-13T18:19:57,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fda96cf025dc,38923,1731521996390 2024-11-13T18:19:57,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:19:57,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:19:57,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:19:57,774 WARN [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:19:57,775 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fda96cf025dc,38923,1731521996390 2024-11-13T18:19:57,777 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T18:19:57,781 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T18:19:57,795 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/hbase.id] with ID: f4366125-239e-4c61-aa45-3d6fdd8fac51 2024-11-13T18:19:57,795 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/.tmp/hbase.id 2024-11-13T18:19:57,808 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/hbase.id] 2024-11-13T18:19:57,882 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:19:57,890 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T18:19:57,918 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-13T18:19:57,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:57,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:57,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:57,963 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:19:57,968 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T18:19:58,017 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:19:58,024 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:19:58,128 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store 2024-11-13T18:19:58,161 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T18:19:58,166 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:19:58,168 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:19:58,169 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:19:58,169 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:19:58,171 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:19:58,171 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:19:58,172 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:19:58,172 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731521998168Disabling compacts and flushes for region at 1731521998168Disabling writes for close at 1731521998171 (+3 ms)Writing region close event to WAL at 1731521998172 (+1 ms)Closed at 1731521998172 2024-11-13T18:19:58,173 WARN [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/.initializing 2024-11-13T18:19:58,173 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/WALs/fda96cf025dc,38923,1731521996390 2024-11-13T18:19:58,203 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:19:58,224 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C38923%2C1731521996390, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/WALs/fda96cf025dc,38923,1731521996390, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/oldWALs, maxLogs=10 2024-11-13T18:19:58,283 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/WALs/fda96cf025dc,38923,1731521996390/fda96cf025dc%2C38923%2C1731521996390.1731521998231 2024-11-13T18:19:58,289 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T18:19:58,290 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:19:58,293 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:19:58,300 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,301 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T18:19:58,432 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:19:58,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:19:58,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T18:19:58,458 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:19:58,458 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:19:58,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,465 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T18:19:58,466 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:19:58,466 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:19:58,467 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,473 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T18:19:58,473 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:19:58,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:19:58,477 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,481 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,482 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,487 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,488 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,494 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:19:58,497 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:19:58,508 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:19:58,509 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71943904, jitterRate=0.07204771041870117}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:19:58,521 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731521998321Initializing all the Stores at 1731521998322 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521998323 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521998329 (+6 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521998329Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521998330 (+1 ms)Cleaning up temporary data from old regions at 1731521998490 (+160 ms)Region opened successfully at 1731521998521 (+31 ms) 2024-11-13T18:19:58,522 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T18:19:58,568 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1af41e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:19:58,616 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T18:19:58,633 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T18:19:58,633 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T18:19:58,638 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:19:58,639 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T18:19:58,658 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 18 msec 2024-11-13T18:19:58,658 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T18:19:58,701 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T18:19:58,715 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T18:19:58,718 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T18:19:58,722 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T18:19:58,724 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T18:19:58,726 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T18:19:58,730 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T18:19:58,741 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T18:19:58,744 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T18:19:58,747 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T18:19:58,755 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T18:19:58,779 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T18:19:58,784 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T18:19:58,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:19:58,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:19:58,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:58,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:19:58,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:58,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:58,811 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fda96cf025dc,38923,1731521996390, sessionid=0x1003a5ef7540000, setting cluster-up flag (Was=false) 2024-11-13T18:19:58,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:58,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:58,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:58,843 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T18:19:58,846 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda96cf025dc,38923,1731521996390 2024-11-13T18:19:58,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:58,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:58,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:58,862 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T18:19:58,865 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda96cf025dc,38923,1731521996390 2024-11-13T18:19:58,873 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T18:19:58,881 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(746): ClusterId : f4366125-239e-4c61-aa45-3d6fdd8fac51 2024-11-13T18:19:58,881 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(746): ClusterId : f4366125-239e-4c61-aa45-3d6fdd8fac51 2024-11-13T18:19:58,884 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:19:58,887 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:19:58,903 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:19:58,904 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:19:58,908 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:19:58,908 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:19:58,910 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:19:58,912 DEBUG [RS:0;fda96cf025dc:40065 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ff325d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:19:58,912 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:19:58,913 DEBUG [RS:1;fda96cf025dc:40625 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bbc5953, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:19:58,941 DEBUG [RS:1;fda96cf025dc:40625 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fda96cf025dc:40625 2024-11-13T18:19:58,944 DEBUG [RS:0;fda96cf025dc:40065 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fda96cf025dc:40065 2024-11-13T18:19:58,946 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:19:58,946 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:19:58,946 DEBUG [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:19:58,946 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:19:58,947 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:19:58,947 DEBUG [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:19:58,950 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,38923,1731521996390 with port=40625, startcode=1731521997570 2024-11-13T18:19:58,957 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,38923,1731521996390 with port=40065, startcode=1731521997373 2024-11-13T18:19:58,967 DEBUG [RS:1;fda96cf025dc:40625 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:19:58,967 DEBUG [RS:0;fda96cf025dc:40065 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:19:58,986 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:19:59,011 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T18:19:59,024 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T18:19:59,038 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fda96cf025dc,38923,1731521996390 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T18:19:59,059 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:19:59,059 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:19:59,059 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:19:59,060 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:19:59,060 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fda96cf025dc:0, corePoolSize=10, maxPoolSize=10 2024-11-13T18:19:59,060 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,060 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:19:59,060 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,124 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:19:59,124 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44655, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:19:59,124 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35555, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:19:59,124 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T18:19:59,131 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:19:59,131 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731522029131 2024-11-13T18:19:59,131 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T18:19:59,132 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38923 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:19:59,137 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T18:19:59,138 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T18:19:59,141 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38923 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:19:59,143 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T18:19:59,143 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T18:19:59,146 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T18:19:59,146 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T18:19:59,163 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,169 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T18:19:59,171 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T18:19:59,172 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T18:19:59,183 DEBUG [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T18:19:59,183 DEBUG [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T18:19:59,183 WARN [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T18:19:59,183 WARN [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T18:19:59,184 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T18:19:59,185 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T18:19:59,188 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731521999186,5,FailOnTimeoutGroup] 2024-11-13T18:19:59,199 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731521999189,5,FailOnTimeoutGroup] 2024-11-13T18:19:59,200 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,200 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T18:19:59,201 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,202 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,213 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T18:19:59,214 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15 2024-11-13T18:19:59,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:19:59,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T18:19:59,238 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:19:59,238 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:19:59,239 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:19:59,239 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T18:19:59,249 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:19:59,249 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:19:59,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:19:59,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T18:19:59,255 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:19:59,255 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:19:59,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:19:59,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T18:19:59,261 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:19:59,261 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:19:59,262 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:19:59,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:19:59,263 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740 2024-11-13T18:19:59,263 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740 2024-11-13T18:19:59,264 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:19:59,264 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:19:59,264 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:19:59,266 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:19:59,276 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:19:59,277 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73174049, jitterRate=0.0903782993555069}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:19:59,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731521999230Initializing all the Stores at 1731521999230Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521999231 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521999231Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521999231Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521999231Cleaning up temporary data from old regions at 1731521999264 (+33 ms)Region opened successfully at 1731521999280 (+16 ms) 2024-11-13T18:19:59,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:19:59,281 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:19:59,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:19:59,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:19:59,282 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:19:59,283 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:19:59,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731521999281Disabling compacts and flushes for region at 1731521999281Disabling writes for close at 1731521999282 (+1 ms)Writing region close event to WAL at 1731521999283 (+1 ms)Closed at 1731521999283 2024-11-13T18:19:59,285 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,38923,1731521996390 with port=40065, startcode=1731521997373 2024-11-13T18:19:59,285 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,38923,1731521996390 with port=40625, startcode=1731521997570 2024-11-13T18:19:59,287 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:19:59,287 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T18:19:59,287 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38923 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,40625,1731521997570 2024-11-13T18:19:59,291 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38923 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,40625,1731521997570 2024-11-13T18:19:59,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T18:19:59,301 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38923 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,40065,1731521997373 2024-11-13T18:19:59,301 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38923 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,40065,1731521997373 2024-11-13T18:19:59,303 DEBUG [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15 2024-11-13T18:19:59,303 DEBUG [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T18:19:59,303 DEBUG [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:19:59,307 DEBUG [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15 2024-11-13T18:19:59,307 DEBUG [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T18:19:59,307 DEBUG [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:19:59,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:19:59,313 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:19:59,314 DEBUG [RS:1;fda96cf025dc:40625 {}] zookeeper.ZKUtil(111): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,40625,1731521997570 2024-11-13T18:19:59,315 WARN [RS:1;fda96cf025dc:40625 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:19:59,315 DEBUG [RS:0;fda96cf025dc:40065 {}] zookeeper.ZKUtil(111): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,40065,1731521997373 2024-11-13T18:19:59,315 INFO [RS:1;fda96cf025dc:40625 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:19:59,315 WARN [RS:0;fda96cf025dc:40065 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:19:59,315 DEBUG [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40625,1731521997570 2024-11-13T18:19:59,315 INFO [RS:0;fda96cf025dc:40065 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:19:59,315 DEBUG [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40065,1731521997373 2024-11-13T18:19:59,318 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,40065,1731521997373] 2024-11-13T18:19:59,318 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,40625,1731521997570] 2024-11-13T18:19:59,318 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T18:19:59,357 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:19:59,357 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:19:59,378 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:19:59,380 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:19:59,385 INFO [RS:0;fda96cf025dc:40065 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:19:59,385 INFO [RS:1;fda96cf025dc:40625 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:19:59,385 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,385 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,386 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:19:59,386 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:19:59,391 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:19:59,393 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,393 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,393 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,394 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,394 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,394 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,394 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:19:59,395 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,395 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,395 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,395 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,396 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,396 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,396 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:19:59,396 DEBUG [RS:1;fda96cf025dc:40625 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:19:59,405 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:19:59,408 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,408 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,409 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,409 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,409 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,409 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,409 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:19:59,410 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,410 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,410 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,410 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,410 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,411 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:19:59,411 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:19:59,411 DEBUG [RS:0;fda96cf025dc:40065 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:19:59,419 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,419 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,419 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,419 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,419 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,419 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,419 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,419 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,419 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,419 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,40065,1731521997373-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:19:59,419 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,419 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,40625,1731521997570-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:19:59,451 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:19:59,451 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:19:59,454 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,40625,1731521997570-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,454 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,454 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.Replication(171): fda96cf025dc,40625,1731521997570 started 2024-11-13T18:19:59,456 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,40065,1731521997373-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,457 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,458 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.Replication(171): fda96cf025dc,40065,1731521997373 started 2024-11-13T18:19:59,471 WARN [fda96cf025dc:38923 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T18:19:59,483 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,484 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,40625,1731521997570, RpcServer on fda96cf025dc/172.17.0.3:40625, sessionid=0x1003a5ef7540002 2024-11-13T18:19:59,485 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:19:59,485 DEBUG [RS:1;fda96cf025dc:40625 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,40625,1731521997570 2024-11-13T18:19:59,485 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,40625,1731521997570' 2024-11-13T18:19:59,486 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:19:59,487 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:19:59,487 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:19:59,488 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,40065,1731521997373, RpcServer on fda96cf025dc/172.17.0.3:40065, sessionid=0x1003a5ef7540001 2024-11-13T18:19:59,488 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:19:59,488 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:19:59,488 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:19:59,489 DEBUG [RS:1;fda96cf025dc:40625 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,40625,1731521997570 2024-11-13T18:19:59,489 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,40625,1731521997570' 2024-11-13T18:19:59,489 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:19:59,490 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:19:59,491 DEBUG [RS:1;fda96cf025dc:40625 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:19:59,491 INFO [RS:1;fda96cf025dc:40625 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:19:59,492 INFO [RS:1;fda96cf025dc:40625 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:19:59,488 DEBUG [RS:0;fda96cf025dc:40065 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,40065,1731521997373 2024-11-13T18:19:59,492 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,40065,1731521997373' 2024-11-13T18:19:59,493 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:19:59,494 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:19:59,495 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:19:59,495 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:19:59,495 DEBUG [RS:0;fda96cf025dc:40065 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,40065,1731521997373 2024-11-13T18:19:59,495 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,40065,1731521997373' 2024-11-13T18:19:59,495 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:19:59,497 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:19:59,498 DEBUG [RS:0;fda96cf025dc:40065 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:19:59,499 INFO [RS:0;fda96cf025dc:40065 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:19:59,500 INFO [RS:0;fda96cf025dc:40065 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:19:59,597 INFO [RS:1;fda96cf025dc:40625 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:19:59,599 INFO [RS:1;fda96cf025dc:40625 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fda96cf025dc%2C40625%2C1731521997570, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40625,1731521997570, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs, maxLogs=55 2024-11-13T18:19:59,603 INFO [RS:0;fda96cf025dc:40065 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:19:59,604 INFO [RS:0;fda96cf025dc:40065 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fda96cf025dc%2C40065%2C1731521997373, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40065,1731521997373, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs, maxLogs=55 2024-11-13T18:19:59,630 INFO [RS:1;fda96cf025dc:40625 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40625,1731521997570/fda96cf025dc%2C40625%2C1731521997570.1731521999602 2024-11-13T18:19:59,631 INFO [RS:0;fda96cf025dc:40065 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40065,1731521997373/fda96cf025dc%2C40065%2C1731521997373.1731521999607 2024-11-13T18:19:59,631 DEBUG [RS:0;fda96cf025dc:40065 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T18:19:59,631 DEBUG [RS:1;fda96cf025dc:40625 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T18:19:59,725 DEBUG [fda96cf025dc:38923 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T18:19:59,739 DEBUG [fda96cf025dc:38923 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:19:59,742 DEBUG [fda96cf025dc:38923 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:19:59,742 DEBUG [fda96cf025dc:38923 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:19:59,742 DEBUG [fda96cf025dc:38923 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:19:59,742 DEBUG [fda96cf025dc:38923 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:19:59,742 INFO [fda96cf025dc:38923 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:19:59,743 INFO [fda96cf025dc:38923 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:19:59,743 DEBUG [fda96cf025dc:38923 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:19:59,752 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fda96cf025dc,40065,1731521997373 2024-11-13T18:19:59,761 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,40065,1731521997373, state=OPENING 2024-11-13T18:19:59,768 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T18:19:59,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:59,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:59,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:59,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:19:59,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:19:59,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:19:59,775 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:19:59,778 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,40065,1731521997373}] 2024-11-13T18:19:59,970 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:19:59,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59449, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:20:00,002 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T18:20:00,003 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:20:00,005 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T18:20:00,007 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fda96cf025dc%2C40065%2C1731521997373.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40065,1731521997373, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs, maxLogs=55 2024-11-13T18:20:00,044 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40065,1731521997373/fda96cf025dc%2C40065%2C1731521997373.meta.1731522000009.meta 2024-11-13T18:20:00,044 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T18:20:00,045 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:20:00,047 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T18:20:00,051 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T18:20:00,057 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T18:20:00,063 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T18:20:00,064 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:00,064 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T18:20:00,065 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T18:20:00,068 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T18:20:00,069 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:20:00,069 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:00,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:00,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T18:20:00,071 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:20:00,071 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:00,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:00,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T18:20:00,073 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:20:00,073 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:00,074 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:00,074 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T18:20:00,075 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:20:00,075 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:00,076 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:00,078 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:20:00,080 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740 2024-11-13T18:20:00,081 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740 2024-11-13T18:20:00,081 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:20:00,081 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:20:00,082 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:20:00,083 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:20:00,084 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59671421, jitterRate=-0.1108265370130539}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:20:00,084 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T18:20:00,086 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731522000066Writing region info on filesystem at 1731522000066Initializing all the Stores at 1731522000066Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522000066Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522000068 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522000068Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522000068Cleaning up temporary data from old regions at 1731522000081 (+13 ms)Running coprocessor post-open hooks at 1731522000084 (+3 ms)Region opened successfully at 1731522000086 (+2 ms) 2024-11-13T18:20:00,093 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731521999954 2024-11-13T18:20:00,120 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,40065,1731521997373 2024-11-13T18:20:00,119 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T18:20:00,120 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T18:20:00,122 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,40065,1731521997373, state=OPEN 2024-11-13T18:20:00,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:20:00,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:20:00,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:20:00,125 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:00,125 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:00,125 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:00,126 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fda96cf025dc,40065,1731521997373 2024-11-13T18:20:00,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T18:20:00,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,40065,1731521997373 in 350 msec 2024-11-13T18:20:00,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T18:20:00,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 837 msec 2024-11-13T18:20:00,135 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:20:00,136 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T18:20:00,158 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:20:00,160 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,40065,1731521997373, seqNum=-1] 2024-11-13T18:20:00,180 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:20:00,185 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38365, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:20:00,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2890 sec 2024-11-13T18:20:00,208 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731522000208, completionTime=-1 2024-11-13T18:20:00,211 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-13T18:20:00,212 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:20:00,244 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T18:20:00,244 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731522060244 2024-11-13T18:20:00,244 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731522120244 2024-11-13T18:20:00,244 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-13T18:20:00,246 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:20:00,252 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,38923,1731521996390-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:00,253 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,38923,1731521996390-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:00,253 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,38923,1731521996390-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:00,255 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fda96cf025dc:38923, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:00,255 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:00,257 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:00,264 DEBUG [master/fda96cf025dc:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T18:20:00,286 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.511sec 2024-11-13T18:20:00,288 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T18:20:00,289 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T18:20:00,290 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T18:20:00,291 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T18:20:00,291 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T18:20:00,291 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,38923,1731521996390-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:20:00,292 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,38923,1731521996390-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T18:20:00,293 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T18:20:00,294 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T18:20:00,294 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,38923,1731521996390-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:00,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0b20dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:20:00,395 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T18:20:00,395 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T18:20:00,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,38923,-1 for getting cluster id 2024-11-13T18:20:00,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:20:00,413 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4366125-239e-4c61-aa45-3d6fdd8fac51' 2024-11-13T18:20:00,416 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:20:00,416 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4366125-239e-4c61-aa45-3d6fdd8fac51" 2024-11-13T18:20:00,418 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cadbcd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:20:00,419 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,38923,-1] 2024-11-13T18:20:00,421 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:20:00,423 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:00,424 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:20:00,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:20:00,427 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:20:00,434 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,40065,1731521997373, seqNum=-1] 2024-11-13T18:20:00,435 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:20:00,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:20:00,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fda96cf025dc,38923,1731521996390 2024-11-13T18:20:00,462 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:20:00,467 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fda96cf025dc,38923,1731521996390 2024-11-13T18:20:00,470 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6628a5c8 2024-11-13T18:20:00,471 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:20:00,490 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:20:00,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38923 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnRegionMove1731522000458', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:20:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38923 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731522000458 2024-11-13T18:20:00,509 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731522000458 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:20:00,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38923 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731522000458" procId is: 4 2024-11-13T18:20:00,513 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:00,514 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731522000458 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:20:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:20:00,532 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731522000458-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e17f6c65cc09b5aa0d090b1afb0ee9b8, NAME => 'testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731522000458', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15 2024-11-13T18:20:00,540 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731522000458-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:00,541 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731522000458-pool-0 {}] regionserver.HRegion(1722): Closing e17f6c65cc09b5aa0d090b1afb0ee9b8, disabling compactions & flushes 2024-11-13T18:20:00,541 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731522000458-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:00,541 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731522000458-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:00,541 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731522000458-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. after waiting 0 ms 2024-11-13T18:20:00,541 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731522000458-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:00,541 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731522000458-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:00,541 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731522000458-pool-0 {}] regionserver.HRegion(1676): Region close journal for e17f6c65cc09b5aa0d090b1afb0ee9b8: Waiting for close lock at 1731522000541Disabling compacts and flushes for region at 1731522000541Disabling writes for close at 1731522000541Writing region close event to WAL at 1731522000541Closed at 1731522000541 2024-11-13T18:20:00,543 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731522000458 execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:20:00,548 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731522000543"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731522000543"}]},"ts":"1731522000543"} 2024-11-13T18:20:00,552 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:20:00,553 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731522000458 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:20:00,556 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731522000458","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522000553"}]},"ts":"1731522000553"} 2024-11-13T18:20:00,560 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731522000458, state=ENABLING in hbase:meta 2024-11-13T18:20:00,561 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:20:00,561 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:20:00,562 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:20:00,562 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:20:00,562 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:20:00,562 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:20:00,562 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:20:00,562 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:20:00,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731522000458, region=e17f6c65cc09b5aa0d090b1afb0ee9b8, ASSIGN}] 2024-11-13T18:20:00,564 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731522000458, region=e17f6c65cc09b5aa0d090b1afb0ee9b8, ASSIGN 2024-11-13T18:20:00,566 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731522000458, region=e17f6c65cc09b5aa0d090b1afb0ee9b8, ASSIGN; state=OFFLINE, location=fda96cf025dc,40625,1731521997570; forceNewPlan=false, retain=false 2024-11-13T18:20:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:20:00,719 INFO [fda96cf025dc:38923 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T18:20:00,720 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e17f6c65cc09b5aa0d090b1afb0ee9b8, regionState=OPENING, regionLocation=fda96cf025dc,40625,1731521997570 2024-11-13T18:20:00,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731522000458, region=e17f6c65cc09b5aa0d090b1afb0ee9b8, ASSIGN because future has completed 2024-11-13T18:20:00,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e17f6c65cc09b5aa0d090b1afb0ee9b8, server=fda96cf025dc,40625,1731521997570}] 2024-11-13T18:20:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:20:00,884 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:20:00,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39629, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:20:00,889 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:00,890 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e17f6c65cc09b5aa0d090b1afb0ee9b8, NAME => 'testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:20:00,891 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731522000458 e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:00,891 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:00,891 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:00,892 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:00,895 INFO [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:00,901 INFO [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e17f6c65cc09b5aa0d090b1afb0ee9b8 columnFamilyName family 2024-11-13T18:20:00,902 DEBUG [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:00,902 INFO [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] regionserver.HStore(327): Store=e17f6c65cc09b5aa0d090b1afb0ee9b8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:20:00,903 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:00,904 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:00,904 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:00,904 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:00,905 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:00,906 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:00,917 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:20:00,917 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e17f6c65cc09b5aa0d090b1afb0ee9b8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62068092, jitterRate=-0.07511335611343384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:20:00,917 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:00,919 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e17f6c65cc09b5aa0d090b1afb0ee9b8: Running coprocessor pre-open hook at 1731522000892Writing region info on filesystem at 1731522000892Initializing all the Stores at 1731522000892Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522000893 (+1 ms)Cleaning up temporary data from old regions at 1731522000905 (+12 ms)Running coprocessor post-open hooks at 1731522000918 (+13 ms)Region opened successfully at 1731522000919 (+1 ms) 2024-11-13T18:20:00,920 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8., pid=6, masterSystemTime=1731522000883 2024-11-13T18:20:00,924 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:00,924 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:00,925 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e17f6c65cc09b5aa0d090b1afb0ee9b8, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,40625,1731521997570 2024-11-13T18:20:00,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e17f6c65cc09b5aa0d090b1afb0ee9b8, server=fda96cf025dc,40625,1731521997570 because future has completed 2024-11-13T18:20:00,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T18:20:00,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e17f6c65cc09b5aa0d090b1afb0ee9b8, server=fda96cf025dc,40625,1731521997570 in 205 msec 2024-11-13T18:20:00,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T18:20:00,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731522000458, region=e17f6c65cc09b5aa0d090b1afb0ee9b8, ASSIGN in 370 msec 2024-11-13T18:20:00,937 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731522000458 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:20:00,938 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731522000458","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522000937"}]},"ts":"1731522000937"} 2024-11-13T18:20:00,941 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731522000458, state=ENABLED in hbase:meta 2024-11-13T18:20:00,942 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731522000458 execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:20:00,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731522000458 in 442 msec 2024-11-13T18:20:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:20:01,153 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731522000458 get assigned. Timeout = 60000ms 2024-11-13T18:20:01,153 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731522000458 completed 2024-11-13T18:20:01,156 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:20:01,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731522000458 assigned to meta. Checking AM states. 2024-11-13T18:20:01,162 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:20:01,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731522000458 assigned. 2024-11-13T18:20:01,172 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731522000458', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8., hostname=fda96cf025dc,40625,1731521997570, seqNum=2] 2024-11-13T18:20:01,173 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:20:01,176 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:20:01,187 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e17f6c65cc09b5aa0d090b1afb0ee9b8 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-13T18:20:01,222 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e1611bb3bb504f90b706ba823563627a 2024-11-13T18:20:01,267 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/.tmp/family/e1611bb3bb504f90b706ba823563627a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-13T18:20:01,272 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/.tmp/family/e1611bb3bb504f90b706ba823563627a 2024-11-13T18:20:01,310 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/.tmp/family/e1611bb3bb504f90b706ba823563627a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a 2024-11-13T18:20:01,312 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a, delay=932 ms 2024-11-13T18:20:01,313 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a, entries=2, sequenceid=6, filesize=4.9 K 2024-11-13T18:20:01,320 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for e17f6c65cc09b5aa0d090b1afb0ee9b8 in 132ms, sequenceid=6, compaction requested=false 2024-11-13T18:20:01,321 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731522000458' 2024-11-13T18:20:01,324 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e17f6c65cc09b5aa0d090b1afb0ee9b8: 2024-11-13T18:20:02,246 DEBUG [hfile-prefetch-1731522001312 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key e1611bb3bb504f90b706ba823563627a_0. 2024-11-13T18:20:02,250 DEBUG [hfile-prefetch-1731522001312 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key e1611bb3bb504f90b706ba823563627a_117. 2024-11-13T18:20:02,251 DEBUG [hfile-prefetch-1731522001312 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a, with total blocks 2, and data blocks 1 2024-11-13T18:20:02,251 DEBUG [hfile-prefetch-1731522001312 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-13T18:20:02,251 DEBUG [hfile-prefetch-1731522001312 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-13T18:20:02,252 DEBUG [hfile-prefetch-1731522001312 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a. Will sleep 100ms and try the verification again. 2024-11-13T18:20:02,294 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-13T18:20:02,294 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-13T18:20:02,295 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T18:20:02,295 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T18:20:02,314 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T18:20:02,314 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731522002294 2024-11-13T18:20:02,314 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-13T18:20:02,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38923 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=e17f6c65cc09b5aa0d090b1afb0ee9b8, source=fda96cf025dc,40625,1731521997570, destination=fda96cf025dc,40065,1731521997373, warming up region on fda96cf025dc,40065,1731521997373 2024-11-13T18:20:02,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38923 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=e17f6c65cc09b5aa0d090b1afb0ee9b8, source=fda96cf025dc,40625,1731521997570, destination=fda96cf025dc,40065,1731521997373, running balancer 2024-11-13T18:20:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38923 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731522000458, region=e17f6c65cc09b5aa0d090b1afb0ee9b8, REOPEN/MOVE 2024-11-13T18:20:02,337 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731522000458, region=e17f6c65cc09b5aa0d090b1afb0ee9b8, REOPEN/MOVE 2024-11-13T18:20:02,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40065 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40065 {}] regionserver.HRegion(7855): Warmup {ENCODED => e17f6c65cc09b5aa0d090b1afb0ee9b8, NAME => 'testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:20:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40065 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:02,339 INFO [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,339 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e17f6c65cc09b5aa0d090b1afb0ee9b8, regionState=CLOSING, regionLocation=fda96cf025dc,40625,1731521997570 2024-11-13T18:20:02,340 INFO [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e17f6c65cc09b5aa0d090b1afb0ee9b8 columnFamilyName family 2024-11-13T18:20:02,340 DEBUG [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:02,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731522000458, region=e17f6c65cc09b5aa0d090b1afb0ee9b8, REOPEN/MOVE because future has completed 2024-11-13T18:20:02,344 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-13T18:20:02,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e17f6c65cc09b5aa0d090b1afb0ee9b8, server=fda96cf025dc,40625,1731521997570}] 2024-11-13T18:20:02,344 DEBUG [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a 2024-11-13T18:20:02,344 INFO [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] regionserver.HStore(327): Store=e17f6c65cc09b5aa0d090b1afb0ee9b8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:20:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40065 {}] regionserver.HRegion(1722): Closing e17f6c65cc09b5aa0d090b1afb0ee9b8, disabling compactions & flushes 2024-11-13T18:20:02,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40065 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40065 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40065 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. after waiting 0 ms 2024-11-13T18:20:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40065 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,347 DEBUG [StoreFileCloser-e17f6c65cc09b5aa0d090b1afb0ee9b8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a 2024-11-13T18:20:02,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40065 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40065 {}] regionserver.HRegion(1676): Region close journal for e17f6c65cc09b5aa0d090b1afb0ee9b8: Waiting for close lock at 1731522002344Disabling compacts and flushes for region at 1731522002344Disabling writes for close at 1731522002345 (+1 ms)Writing region close event to WAL at 1731522002347 (+2 ms)Closed at 1731522002347 2024-11-13T18:20:02,349 WARN [hfile-prefetch-1731522001312 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-13T18:20:02,350 DEBUG [hfile-prefetch-1731522001312 {}] hfile.PrefetchExecutor(121): Prefetch completed for e1611bb3bb504f90b706ba823563627a 2024-11-13T18:20:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38923 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T18:20:02,502 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,502 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-13T18:20:02,503 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing e17f6c65cc09b5aa0d090b1afb0ee9b8, disabling compactions & flushes 2024-11-13T18:20:02,503 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,503 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,503 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. after waiting 0 ms 2024-11-13T18:20:02,503 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,505 DEBUG [StoreFileCloser-e17f6c65cc09b5aa0d090b1afb0ee9b8-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file e1611bb3bb504f90b706ba823563627a, starting offset: 0, end offset: 9223372036854775807 2024-11-13T18:20:02,506 DEBUG [StoreFileCloser-e17f6c65cc09b5aa0d090b1afb0ee9b8-family-1 {}] bucket.BucketCache(857): removed key e1611bb3bb504f90b706ba823563627a_0 from back map with offset lock 2094080 in the evict process 2024-11-13T18:20:02,513 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-13T18:20:02,514 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-13T18:20:02,514 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-13T18:20:02,514 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T18:20:02,514 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T18:20:02,514 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,515 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for e17f6c65cc09b5aa0d090b1afb0ee9b8: Waiting for close lock at 1731522002503Running coprocessor pre-close hooks at 1731522002503Disabling compacts and flushes for region at 1731522002503Disabling writes for close at 1731522002503Writing region close event to WAL at 1731522002507 (+4 ms)Running coprocessor post-close hooks at 1731522002513 (+6 ms)Closed at 1731522002514 (+1 ms) 2024-11-13T18:20:02,515 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding e17f6c65cc09b5aa0d090b1afb0ee9b8 move to fda96cf025dc,40065,1731521997373 record at close sequenceid=6 2024-11-13T18:20:02,519 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T18:20:02,519 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731522002514 2024-11-13T18:20:02,519 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,519 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e17f6c65cc09b5aa0d090b1afb0ee9b8, regionState=CLOSED 2024-11-13T18:20:02,519 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-13T18:20:02,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e17f6c65cc09b5aa0d090b1afb0ee9b8, server=fda96cf025dc,40625,1731521997570 because future has completed 2024-11-13T18:20:02,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T18:20:02,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure e17f6c65cc09b5aa0d090b1afb0ee9b8, server=fda96cf025dc,40625,1731521997570 in 180 msec 2024-11-13T18:20:02,528 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731522000458, region=e17f6c65cc09b5aa0d090b1afb0ee9b8, REOPEN/MOVE; state=CLOSED, location=fda96cf025dc,40065,1731521997373; forceNewPlan=false, retain=false 2024-11-13T18:20:02,678 INFO [fda96cf025dc:38923 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T18:20:02,679 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e17f6c65cc09b5aa0d090b1afb0ee9b8, regionState=OPENING, regionLocation=fda96cf025dc,40065,1731521997373 2024-11-13T18:20:02,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731522000458, region=e17f6c65cc09b5aa0d090b1afb0ee9b8, REOPEN/MOVE because future has completed 2024-11-13T18:20:02,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e17f6c65cc09b5aa0d090b1afb0ee9b8, server=fda96cf025dc,40065,1731521997373}] 2024-11-13T18:20:02,840 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,840 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e17f6c65cc09b5aa0d090b1afb0ee9b8, NAME => 'testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:20:02,841 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731522000458 e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,841 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:02,841 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,841 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,842 INFO [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,844 INFO [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e17f6c65cc09b5aa0d090b1afb0ee9b8 columnFamilyName family 2024-11-13T18:20:02,844 DEBUG [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:02,846 DEBUG [StoreFileOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a, delay=977 ms 2024-11-13T18:20:02,848 DEBUG [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a 2024-11-13T18:20:02,849 INFO [StoreOpener-e17f6c65cc09b5aa0d090b1afb0ee9b8-1 {}] regionserver.HStore(327): Store=e17f6c65cc09b5aa0d090b1afb0ee9b8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:20:02,850 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,851 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,851 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,852 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,852 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,858 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,859 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened e17f6c65cc09b5aa0d090b1afb0ee9b8; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71316052, jitterRate=0.06269198656082153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:20:02,859 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,861 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e17f6c65cc09b5aa0d090b1afb0ee9b8: Running coprocessor pre-open hook at 1731522002841Writing region info on filesystem at 1731522002841Initializing all the Stores at 1731522002842 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522002842Cleaning up temporary data from old regions at 1731522002852 (+10 ms)Running coprocessor post-open hooks at 1731522002859 (+7 ms)Region opened successfully at 1731522002861 (+2 ms) 2024-11-13T18:20:02,862 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8., pid=9, masterSystemTime=1731522002837 2024-11-13T18:20:02,865 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,865 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,866 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e17f6c65cc09b5aa0d090b1afb0ee9b8, regionState=OPEN, openSeqNum=10, regionLocation=fda96cf025dc,40065,1731521997373 2024-11-13T18:20:02,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e17f6c65cc09b5aa0d090b1afb0ee9b8, server=fda96cf025dc,40065,1731521997373 because future has completed 2024-11-13T18:20:02,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T18:20:02,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure e17f6c65cc09b5aa0d090b1afb0ee9b8, server=fda96cf025dc,40065,1731521997373 in 188 msec 2024-11-13T18:20:02,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731522000458, region=e17f6c65cc09b5aa0d090b1afb0ee9b8, REOPEN/MOVE in 538 msec 2024-11-13T18:20:02,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T18:20:02,890 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:20:02,890 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:20:02,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:02,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:02,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:20:02,897 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T18:20:02,897 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-13T18:20:02,897 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fda96cf025dc,38923,1731521996390 2024-11-13T18:20:02,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:20:02,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:02,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:20:02,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:02,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:20:02,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:02,901 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:20:02,901 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:20:02,902 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:20:02,903 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:20:02,904 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:20:02,904 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:20:02,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:02,904 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,40065,1731521997373' ***** 2024-11-13T18:20:02,904 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:20:02,904 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,40625,1731521997570' ***** 2024-11-13T18:20:02,905 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:20:02,905 INFO [RS:1;fda96cf025dc:40625 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T18:20:02,905 INFO [RS:1;fda96cf025dc:40625 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/bucket.persistence 2024-11-13T18:20:02,905 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T18:20:02,905 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T18:20:02,905 INFO [RS:0;fda96cf025dc:40065 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T18:20:02,905 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T18:20:02,905 INFO [RS:0;fda96cf025dc:40065 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/bucket.persistence 2024-11-13T18:20:02,905 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T18:20:02,905 INFO [RS:1;fda96cf025dc:40625 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T18:20:02,906 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T18:20:02,906 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T18:20:02,906 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T18:20:02,906 INFO [RS:0;fda96cf025dc:40065 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T18:20:02,906 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T18:20:02,916 DEBUG [RS:1;fda96cf025dc:40625 {}] bucket.BucketCache(1488): Thread RS:1;fda96cf025dc:40625 started persisting bucket cache to file 2024-11-13T18:20:02,916 DEBUG [RS:1;fda96cf025dc:40625 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T18:20:02,916 DEBUG [RS:1;fda96cf025dc:40625 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T18:20:02,916 DEBUG [RS:0;fda96cf025dc:40065 {}] bucket.BucketCache(1488): Thread RS:0;fda96cf025dc:40065 started persisting bucket cache to file 2024-11-13T18:20:02,917 DEBUG [RS:0;fda96cf025dc:40065 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T18:20:02,917 DEBUG [RS:0;fda96cf025dc:40065 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T18:20:02,921 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:20:02,921 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:20:02,924 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:20:02,924 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:20:02,925 WARN [Thread-95 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-13T18:20:02,925 DEBUG [RS:1;fda96cf025dc:40625 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T18:20:02,925 DEBUG [RS:1;fda96cf025dc:40625 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731522002916 2024-11-13T18:20:02,925 DEBUG [RS:0;fda96cf025dc:40065 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T18:20:02,925 DEBUG [RS:1;fda96cf025dc:40625 {}] bucket.BucketCache(1510): Thread RS:1;fda96cf025dc:40625 finished persisting bucket cache to file, renaming 2024-11-13T18:20:02,925 DEBUG [RS:0;fda96cf025dc:40065 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731522002916 2024-11-13T18:20:02,926 DEBUG [RS:0;fda96cf025dc:40065 {}] bucket.BucketCache(1510): Thread RS:0;fda96cf025dc:40065 finished persisting bucket cache to file, renaming 2024-11-13T18:20:02,926 WARN [RS:0;fda96cf025dc:40065 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-13T18:20:02,926 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:20:02,926 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:20:02,926 INFO [RS:1;fda96cf025dc:40625 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:20:02,926 INFO [RS:0;fda96cf025dc:40065 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:20:02,926 INFO [RS:0;fda96cf025dc:40065 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:20:02,926 INFO [RS:1;fda96cf025dc:40625 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:20:02,926 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(959): stopping server fda96cf025dc,40625,1731521997570 2024-11-13T18:20:02,926 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(3091): Received CLOSE for e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,927 INFO [RS:1;fda96cf025dc:40625 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:20:02,927 INFO [RS:1;fda96cf025dc:40625 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fda96cf025dc:40625. 2024-11-13T18:20:02,927 DEBUG [RS:1;fda96cf025dc:40625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:20:02,927 DEBUG [RS:1;fda96cf025dc:40625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:02,927 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(959): stopping server fda96cf025dc,40065,1731521997373 2024-11-13T18:20:02,927 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(976): stopping server fda96cf025dc,40625,1731521997570; all regions closed. 2024-11-13T18:20:02,927 INFO [RS:0;fda96cf025dc:40065 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:20:02,927 INFO [RS:0;fda96cf025dc:40065 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fda96cf025dc:40065. 2024-11-13T18:20:02,927 DEBUG [RS:0;fda96cf025dc:40065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:20:02,927 DEBUG [RS:0;fda96cf025dc:40065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:02,928 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T18:20:02,928 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:20:02,928 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:20:02,928 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e17f6c65cc09b5aa0d090b1afb0ee9b8, disabling compactions & flushes 2024-11-13T18:20:02,928 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:20:02,928 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,928 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T18:20:02,928 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,928 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. after waiting 0 ms 2024-11-13T18:20:02,928 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,928 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T18:20:02,929 DEBUG [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, e17f6c65cc09b5aa0d090b1afb0ee9b8=testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8.} 2024-11-13T18:20:02,929 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T18:20:02,929 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:20:02,929 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:20:02,929 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:20:02,929 DEBUG [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e17f6c65cc09b5aa0d090b1afb0ee9b8 2024-11-13T18:20:02,929 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:20:02,929 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:20:02,931 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-13T18:20:02,931 DEBUG [StoreFileCloser-e17f6c65cc09b5aa0d090b1afb0ee9b8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/family/e1611bb3bb504f90b706ba823563627a 2024-11-13T18:20:02,942 DEBUG [RS:1;fda96cf025dc:40625 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs 2024-11-13T18:20:02,943 INFO [RS:1;fda96cf025dc:40625 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C40625%2C1731521997570:(num 1731521999602) 2024-11-13T18:20:02,943 DEBUG [RS:1;fda96cf025dc:40625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:02,943 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:20:02,943 INFO [RS:1;fda96cf025dc:40625 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:20:02,943 INFO [RS:1;fda96cf025dc:40625 {}] hbase.ChoreService(370): Chore service for: regionserver/fda96cf025dc:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T18:20:02,944 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:20:02,944 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:20:02,944 INFO [regionserver/fda96cf025dc:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:20:02,944 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:20:02,944 INFO [RS:1;fda96cf025dc:40625 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:20:02,944 INFO [RS:1;fda96cf025dc:40625 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40625 2024-11-13T18:20:02,945 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnRegionMove1731522000458/e17f6c65cc09b5aa0d090b1afb0ee9b8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-13T18:20:02,945 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,945 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e17f6c65cc09b5aa0d090b1afb0ee9b8: Waiting for close lock at 1731522002927Running coprocessor pre-close hooks at 1731522002928 (+1 ms)Disabling compacts and flushes for region at 1731522002928Disabling writes for close at 1731522002928Writing region close event to WAL at 1731522002932 (+4 ms)Running coprocessor post-close hooks at 1731522002945 (+13 ms)Closed at 1731522002945 2024-11-13T18:20:02,946 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8. 2024-11-13T18:20:02,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:20:02,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,40625,1731521997570 2024-11-13T18:20:02,950 INFO [RS:1;fda96cf025dc:40625 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:20:02,951 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,40625,1731521997570] 2024-11-13T18:20:02,953 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f635b847dd2b4f4ab3d14ff50cfc7b60 2024-11-13T18:20:02,953 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,40625,1731521997570 already deleted, retry=false 2024-11-13T18:20:02,954 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fda96cf025dc,40625,1731521997570 expired; onlineServers=1 2024-11-13T18:20:02,982 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/info/f635b847dd2b4f4ab3d14ff50cfc7b60 is 205, key is testBlockEvictionOnRegionMove1731522000458,,1731522000492.e17f6c65cc09b5aa0d090b1afb0ee9b8./info:regioninfo/1731522002866/Put/seqid=0 2024-11-13T18:20:02,983 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/info/f635b847dd2b4f4ab3d14ff50cfc7b60 2024-11-13T18:20:02,997 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 92cf812f0b684bf3963478c305fd5932 2024-11-13T18:20:03,013 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/ns/92cf812f0b684bf3963478c305fd5932 is 43, key is default/ns:d/1731522000190/Put/seqid=0 2024-11-13T18:20:03,013 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/ns/92cf812f0b684bf3963478c305fd5932 2024-11-13T18:20:03,026 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c0a2cf7e59bc4df395f21f9c4c43e4ac 2024-11-13T18:20:03,047 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/table/c0a2cf7e59bc4df395f21f9c4c43e4ac is 78, key is testBlockEvictionOnRegionMove1731522000458/table:state/1731522000937/Put/seqid=0 2024-11-13T18:20:03,049 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/table/c0a2cf7e59bc4df395f21f9c4c43e4ac 2024-11-13T18:20:03,052 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/info/f635b847dd2b4f4ab3d14ff50cfc7b60 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/f635b847dd2b4f4ab3d14ff50cfc7b60 2024-11-13T18:20:03,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:03,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40625-0x1003a5ef7540002, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:03,054 INFO [RS:1;fda96cf025dc:40625 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:20:03,054 INFO [RS:1;fda96cf025dc:40625 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,40625,1731521997570; zookeeper connection closed. 2024-11-13T18:20:03,054 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@11d2b18f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@11d2b18f 2024-11-13T18:20:03,054 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/f635b847dd2b4f4ab3d14ff50cfc7b60, delay=988 ms 2024-11-13T18:20:03,056 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/f635b847dd2b4f4ab3d14ff50cfc7b60, entries=15, sequenceid=15, filesize=7.6 K 2024-11-13T18:20:03,057 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/ns/92cf812f0b684bf3963478c305fd5932 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/ns/92cf812f0b684bf3963478c305fd5932 2024-11-13T18:20:03,059 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/ns/92cf812f0b684bf3963478c305fd5932, delay=977 ms 2024-11-13T18:20:03,061 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/ns/92cf812f0b684bf3963478c305fd5932, entries=2, sequenceid=15, filesize=5.0 K 2024-11-13T18:20:03,061 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/table/c0a2cf7e59bc4df395f21f9c4c43e4ac as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/table/c0a2cf7e59bc4df395f21f9c4c43e4ac 2024-11-13T18:20:03,063 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/table/c0a2cf7e59bc4df395f21f9c4c43e4ac, delay=917 ms 2024-11-13T18:20:03,065 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/table/c0a2cf7e59bc4df395f21f9c4c43e4ac, entries=2, sequenceid=15, filesize=5.3 K 2024-11-13T18:20:03,066 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 136ms, sequenceid=15, compaction requested=false 2024-11-13T18:20:03,066 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T18:20:03,068 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/f635b847dd2b4f4ab3d14ff50cfc7b60 2024-11-13T18:20:03,071 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/ns/92cf812f0b684bf3963478c305fd5932 2024-11-13T18:20:03,071 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/table/c0a2cf7e59bc4df395f21f9c4c43e4ac 2024-11-13T18:20:03,077 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-13T18:20:03,077 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:20:03,078 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:20:03,078 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731522002929Running coprocessor pre-close hooks at 1731522002929Disabling compacts and flushes for region at 1731522002929Disabling writes for close at 1731522002929Obtaining lock to block concurrent updates at 1731522002931 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1731522002931Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731522002932 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731522002932Flushing 1588230740/info: creating writer at 1731522002932Flushing 1588230740/info: appending metadata at 1731522002981 (+49 ms)Flushing 1588230740/info: closing flushed file at 1731522002981Flushing 1588230740/ns: creating writer at 1731522002987 (+6 ms)Flushing 1588230740/ns: appending metadata at 1731522003012 (+25 ms)Flushing 1588230740/ns: closing flushed file at 1731522003012Flushing 1588230740/table: creating writer at 1731522003017 (+5 ms)Flushing 1588230740/table: appending metadata at 1731522003047 (+30 ms)Flushing 1588230740/table: closing flushed file at 1731522003047Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b9d007b: reopening flushed file at 1731522003052 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e89a6ba: reopening flushed file at 1731522003057 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@725f2ea6: reopening flushed file at 1731522003061 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 136ms, sequenceid=15, compaction requested=false at 1731522003066 (+5 ms)Writing region close event to WAL at 1731522003072 (+6 ms)Running coprocessor post-close hooks at 1731522003077 (+5 ms)Closed at 1731522003078 (+1 ms) 2024-11-13T18:20:03,078 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T18:20:03,129 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(976): stopping server fda96cf025dc,40065,1731521997373; all regions closed. 2024-11-13T18:20:03,132 DEBUG [RS:0;fda96cf025dc:40065 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs 2024-11-13T18:20:03,133 INFO [RS:0;fda96cf025dc:40065 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C40065%2C1731521997373.meta:.meta(num 1731522000009) 2024-11-13T18:20:03,136 DEBUG [RS:0;fda96cf025dc:40065 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs 2024-11-13T18:20:03,136 INFO [RS:0;fda96cf025dc:40065 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C40065%2C1731521997373:(num 1731521999607) 2024-11-13T18:20:03,136 DEBUG [RS:0;fda96cf025dc:40065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:03,136 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:20:03,136 INFO [RS:0;fda96cf025dc:40065 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:20:03,136 INFO [RS:0;fda96cf025dc:40065 {}] hbase.ChoreService(370): Chore service for: regionserver/fda96cf025dc:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T18:20:03,137 INFO [RS:0;fda96cf025dc:40065 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:20:03,137 INFO [regionserver/fda96cf025dc:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:20:03,137 INFO [RS:0;fda96cf025dc:40065 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40065 2024-11-13T18:20:03,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:20:03,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,40065,1731521997373 2024-11-13T18:20:03,140 INFO [RS:0;fda96cf025dc:40065 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:20:03,143 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,40065,1731521997373] 2024-11-13T18:20:03,145 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,40065,1731521997373 already deleted, retry=false 2024-11-13T18:20:03,145 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fda96cf025dc,40065,1731521997373 expired; onlineServers=0 2024-11-13T18:20:03,145 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fda96cf025dc,38923,1731521996390' ***** 2024-11-13T18:20:03,145 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T18:20:03,145 INFO [M:0;fda96cf025dc:38923 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:20:03,145 INFO [M:0;fda96cf025dc:38923 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:20:03,145 DEBUG [M:0;fda96cf025dc:38923 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T18:20:03,146 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T18:20:03,146 DEBUG [M:0;fda96cf025dc:38923 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T18:20:03,146 DEBUG [master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731521999186 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731521999186,5,FailOnTimeoutGroup] 2024-11-13T18:20:03,146 DEBUG [master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731521999189 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731521999189,5,FailOnTimeoutGroup] 2024-11-13T18:20:03,146 INFO [M:0;fda96cf025dc:38923 {}] hbase.ChoreService(370): Chore service for: master/fda96cf025dc:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T18:20:03,146 INFO [M:0;fda96cf025dc:38923 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:20:03,146 DEBUG [M:0;fda96cf025dc:38923 {}] master.HMaster(1795): Stopping service threads 2024-11-13T18:20:03,146 INFO [M:0;fda96cf025dc:38923 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T18:20:03,147 INFO [M:0;fda96cf025dc:38923 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:20:03,147 INFO [M:0;fda96cf025dc:38923 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:20:03,147 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T18:20:03,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T18:20:03,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,148 DEBUG [M:0;fda96cf025dc:38923 {}] zookeeper.ZKUtil(347): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T18:20:03,148 WARN [M:0;fda96cf025dc:38923 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T18:20:03,149 INFO [M:0;fda96cf025dc:38923 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/.lastflushedseqids 2024-11-13T18:20:03,161 INFO [M:0;fda96cf025dc:38923 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:20:03,161 INFO [M:0;fda96cf025dc:38923 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T18:20:03,162 DEBUG [M:0;fda96cf025dc:38923 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:20:03,162 INFO [M:0;fda96cf025dc:38923 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:03,162 DEBUG [M:0;fda96cf025dc:38923 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:03,162 DEBUG [M:0;fda96cf025dc:38923 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:20:03,162 DEBUG [M:0;fda96cf025dc:38923 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:03,162 INFO [M:0;fda96cf025dc:38923 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-13T18:20:03,181 DEBUG [M:0;fda96cf025dc:38923 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bb25a1b60fa04929979ecc73b22e0702 2024-11-13T18:20:03,202 DEBUG [M:0;fda96cf025dc:38923 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb25a1b60fa04929979ecc73b22e0702 is 82, key is hbase:meta,,1/info:regioninfo/1731522000119/Put/seqid=0 2024-11-13T18:20:03,204 INFO [M:0;fda96cf025dc:38923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb25a1b60fa04929979ecc73b22e0702 2024-11-13T18:20:03,215 DEBUG [M:0;fda96cf025dc:38923 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 374293019043459a94c71a0ed7c27bd4 2024-11-13T18:20:03,237 DEBUG [M:0;fda96cf025dc:38923 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/374293019043459a94c71a0ed7c27bd4 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731522000943/Put/seqid=0 2024-11-13T18:20:03,238 INFO [M:0;fda96cf025dc:38923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/374293019043459a94c71a0ed7c27bd4 2024-11-13T18:20:03,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:03,243 INFO [RS:0;fda96cf025dc:40065 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:20:03,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40065-0x1003a5ef7540001, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:03,243 INFO [RS:0;fda96cf025dc:40065 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,40065,1731521997373; zookeeper connection closed. 2024-11-13T18:20:03,244 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@348eeb71 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@348eeb71 2024-11-13T18:20:03,244 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-13T18:20:03,247 DEBUG [M:0;fda96cf025dc:38923 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 997126e16c104b668b91181a144610d5 2024-11-13T18:20:03,261 DEBUG [M:0;fda96cf025dc:38923 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/997126e16c104b668b91181a144610d5 is 69, key is fda96cf025dc,40065,1731521997373/rs:state/1731521999301/Put/seqid=0 2024-11-13T18:20:03,262 INFO [M:0;fda96cf025dc:38923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/997126e16c104b668b91181a144610d5 2024-11-13T18:20:03,264 DEBUG [M:0;fda96cf025dc:38923 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb25a1b60fa04929979ecc73b22e0702 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bb25a1b60fa04929979ecc73b22e0702 2024-11-13T18:20:03,265 INFO [M:0;fda96cf025dc:38923 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bb25a1b60fa04929979ecc73b22e0702, entries=8, sequenceid=84, filesize=5.5 K 2024-11-13T18:20:03,266 DEBUG [M:0;fda96cf025dc:38923 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/374293019043459a94c71a0ed7c27bd4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/374293019043459a94c71a0ed7c27bd4 2024-11-13T18:20:03,268 INFO [M:0;fda96cf025dc:38923 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/374293019043459a94c71a0ed7c27bd4, entries=9, sequenceid=84, filesize=6.5 K 2024-11-13T18:20:03,268 DEBUG [M:0;fda96cf025dc:38923 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/997126e16c104b668b91181a144610d5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/997126e16c104b668b91181a144610d5 2024-11-13T18:20:03,269 INFO [M:0;fda96cf025dc:38923 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/997126e16c104b668b91181a144610d5, entries=2, sequenceid=84, filesize=5.1 K 2024-11-13T18:20:03,270 INFO [M:0;fda96cf025dc:38923 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 108ms, sequenceid=84, compaction requested=false 2024-11-13T18:20:03,272 INFO [M:0;fda96cf025dc:38923 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:03,272 DEBUG [M:0;fda96cf025dc:38923 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731522003161Disabling compacts and flushes for region at 1731522003162 (+1 ms)Disabling writes for close at 1731522003162Obtaining lock to block concurrent updates at 1731522003162Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731522003162Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731522003162Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731522003163 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731522003163Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731522003202 (+39 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731522003202Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731522003206 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731522003236 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731522003236Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731522003240 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731522003261 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731522003261Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7955e4fb: reopening flushed file at 1731522003263 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c030262: reopening flushed file at 1731522003266 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c5d24a5: reopening flushed file at 1731522003268 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 108ms, sequenceid=84, compaction requested=false at 1731522003270 (+2 ms)Writing region close event to WAL at 1731522003272 (+2 ms)Closed at 1731522003272 2024-11-13T18:20:03,274 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:20:03,274 INFO [M:0;fda96cf025dc:38923 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T18:20:03,274 INFO [M:0;fda96cf025dc:38923 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38923 2024-11-13T18:20:03,275 INFO [M:0;fda96cf025dc:38923 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:20:03,377 INFO [M:0;fda96cf025dc:38923 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:20:03,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:03,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38923-0x1003a5ef7540000, quorum=127.0.0.1:54759, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:03,383 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T18:20:03,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T18:20:03,403 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:03,412 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/fda96cf025dc:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731522003055 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731522001312 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/fda96cf025dc:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731522003059 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/fda96cf025dc:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731522002847 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/fda96cf025dc:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=627 (was 649), ProcessCount=11 (was 12), AvailableMemoryMB=3814 (was 4372) 2024-11-13T18:20:03,418 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=627, ProcessCount=11, AvailableMemoryMB=3813 2024-11-13T18:20:03,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/hadoop-log-dir so I do NOT create it in target/test-data/19622564-d1f4-1879-8371-43b9632b515b 2024-11-13T18:20:03,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/hadoop-tmp-dir so I do NOT create it in target/test-data/19622564-d1f4-1879-8371-43b9632b515b 2024-11-13T18:20:03,418 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:03,421 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/cluster_0c39cbf8-7e0d-719d-f7fe-7af12f953d66, deleteOnExit=true 2024-11-13T18:20:03,424 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/cluster_0c39cbf8-7e0d-719d-f7fe-7af12f953d66/zookeeper_0, clientPort=56021, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/cluster_0c39cbf8-7e0d-719d-f7fe-7af12f953d66/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/cluster_0c39cbf8-7e0d-719d-f7fe-7af12f953d66/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T18:20:03,426 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56021 2024-11-13T18:20:03,440 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15 with version=8 2024-11-13T18:20:03,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/hbase-staging 2024-11-13T18:20:03,443 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:20:03,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:03,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:03,443 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:20:03,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:03,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:20:03,444 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T18:20:03,444 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:20:03,444 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46067 2024-11-13T18:20:03,446 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46067 connecting to ZooKeeper ensemble=127.0.0.1:56021 2024-11-13T18:20:03,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:460670x0, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:20:03,458 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46067-0x1003a5f145f0000 connected 2024-11-13T18:20:03,487 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:03,487 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:03,490 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:20:03,490 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15, hbase.cluster.distributed=false 2024-11-13T18:20:03,492 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:20:03,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46067 2024-11-13T18:20:03,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46067 2024-11-13T18:20:03,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46067 2024-11-13T18:20:03,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46067 2024-11-13T18:20:03,500 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46067 2024-11-13T18:20:03,526 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:20:03,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:03,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:03,527 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:20:03,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:03,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:20:03,527 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:20:03,527 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:20:03,528 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41733 2024-11-13T18:20:03,532 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41733 connecting to ZooKeeper ensemble=127.0.0.1:56021 2024-11-13T18:20:03,533 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:03,533 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:03,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417330x0, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:20:03,540 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:417330x0, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:20:03,541 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:20:03,542 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.cache 2024-11-13T18:20:03,542 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T18:20:03,542 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T18:20:03,542 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T18:20:03,543 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T18:20:03,543 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T18:20:03,543 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T18:20:03,543 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T18:20:03,544 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.persistence is complete. 2024-11-13T18:20:03,544 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:20:03,544 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41733-0x1003a5f145f0001 connected 2024-11-13T18:20:03,547 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:20:03,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:20:03,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41733 2024-11-13T18:20:03,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41733 2024-11-13T18:20:03,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41733 2024-11-13T18:20:03,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41733 2024-11-13T18:20:03,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41733 2024-11-13T18:20:03,595 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:20:03,596 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:03,596 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:03,596 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:20:03,596 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:03,596 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:20:03,596 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:20:03,596 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:20:03,597 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40711 2024-11-13T18:20:03,599 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40711 connecting to ZooKeeper ensemble=127.0.0.1:56021 2024-11-13T18:20:03,599 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:03,599 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:03,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407110x0, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:20:03,604 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40711-0x1003a5f145f0002 connected 2024-11-13T18:20:03,604 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:20:03,604 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:20:03,606 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.cache 2024-11-13T18:20:03,606 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T18:20:03,606 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T18:20:03,606 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T18:20:03,607 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T18:20:03,607 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T18:20:03,607 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T18:20:03,607 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T18:20:03,607 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:20:03,607 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.persistence is complete. 2024-11-13T18:20:03,608 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:20:03,611 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:20:03,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40711 2024-11-13T18:20:03,617 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40711 2024-11-13T18:20:03,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40711 2024-11-13T18:20:03,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40711 2024-11-13T18:20:03,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40711 2024-11-13T18:20:03,640 DEBUG [M:0;fda96cf025dc:46067 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fda96cf025dc:46067 2024-11-13T18:20:03,640 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fda96cf025dc,46067,1731522003442 2024-11-13T18:20:03,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:20:03,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:20:03,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:20:03,644 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fda96cf025dc,46067,1731522003442 2024-11-13T18:20:03,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:20:03,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:20:03,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,650 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T18:20:03,650 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fda96cf025dc,46067,1731522003442 from backup master directory 2024-11-13T18:20:03,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fda96cf025dc,46067,1731522003442 2024-11-13T18:20:03,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:20:03,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:20:03,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:20:03,653 WARN [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:20:03,654 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fda96cf025dc,46067,1731522003442 2024-11-13T18:20:03,655 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/hbase.id] with ID: f3a3bf4d-9732-4ab1-9e7c-286aef4c2a8a 2024-11-13T18:20:03,655 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/.tmp/hbase.id 2024-11-13T18:20:03,661 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/hbase.id] 2024-11-13T18:20:03,701 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:03,701 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T18:20:03,703 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-13T18:20:03,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,711 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:20:03,712 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T18:20:03,713 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:20:03,734 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store 2024-11-13T18:20:03,740 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:03,740 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:20:03,740 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:03,740 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:03,741 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:20:03,741 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:03,741 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:03,741 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731522003740Disabling compacts and flushes for region at 1731522003740Disabling writes for close at 1731522003741 (+1 ms)Writing region close event to WAL at 1731522003741Closed at 1731522003741 2024-11-13T18:20:03,741 WARN [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/.initializing 2024-11-13T18:20:03,741 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/WALs/fda96cf025dc,46067,1731522003442 2024-11-13T18:20:03,747 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:20:03,756 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C46067%2C1731522003442, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/WALs/fda96cf025dc,46067,1731522003442, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/oldWALs, maxLogs=10 2024-11-13T18:20:03,779 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/WALs/fda96cf025dc,46067,1731522003442/fda96cf025dc%2C46067%2C1731522003442.1731522003759 2024-11-13T18:20:03,779 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T18:20:03,779 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:20:03,780 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:03,780 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,780 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,781 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,786 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T18:20:03,786 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:03,786 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:03,787 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T18:20:03,791 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:03,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:20:03,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,795 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T18:20:03,795 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:03,795 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:20:03,795 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,799 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T18:20:03,799 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:03,799 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:20:03,800 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,800 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,800 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,800 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,800 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,801 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:20:03,801 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:20:03,816 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:20:03,817 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74014830, jitterRate=0.10290691256523132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:20:03,818 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731522003780Initializing all the Stores at 1731522003780Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522003781 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522003781Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522003781Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522003781Cleaning up temporary data from old regions at 1731522003800 (+19 ms)Region opened successfully at 1731522003817 (+17 ms) 2024-11-13T18:20:03,818 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T18:20:03,823 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508cd868, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:20:03,824 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T18:20:03,824 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T18:20:03,825 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T18:20:03,825 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:20:03,825 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T18:20:03,826 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-13T18:20:03,826 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T18:20:03,828 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T18:20:03,829 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T18:20:03,831 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T18:20:03,831 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T18:20:03,832 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T18:20:03,834 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T18:20:03,834 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T18:20:03,836 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T18:20:03,837 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T18:20:03,839 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T18:20:03,841 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T18:20:03,843 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T18:20:03,845 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T18:20:03,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:20:03,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:20:03,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:20:03,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,849 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fda96cf025dc,46067,1731522003442, sessionid=0x1003a5f145f0000, setting cluster-up flag (Was=false) 2024-11-13T18:20:03,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,859 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T18:20:03,860 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda96cf025dc,46067,1731522003442 2024-11-13T18:20:03,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:03,872 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T18:20:03,873 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda96cf025dc,46067,1731522003442 2024-11-13T18:20:03,874 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T18:20:03,876 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:20:03,876 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T18:20:03,876 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T18:20:03,877 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fda96cf025dc,46067,1731522003442 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T18:20:03,881 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:20:03,882 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:20:03,882 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:20:03,882 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:20:03,882 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fda96cf025dc:0, corePoolSize=10, maxPoolSize=10 2024-11-13T18:20:03,882 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,882 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:20:03,882 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,884 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:20:03,884 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T18:20:03,885 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:03,885 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T18:20:03,890 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731522033890 2024-11-13T18:20:03,890 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T18:20:03,890 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T18:20:03,890 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T18:20:03,890 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T18:20:03,890 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T18:20:03,890 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T18:20:03,893 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:03,897 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T18:20:03,897 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T18:20:03,897 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T18:20:03,899 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T18:20:03,899 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T18:20:03,900 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731522003899,5,FailOnTimeoutGroup] 2024-11-13T18:20:03,904 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731522003900,5,FailOnTimeoutGroup] 2024-11-13T18:20:03,904 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:03,904 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T18:20:03,904 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:03,904 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:03,911 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T18:20:03,911 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15 2024-11-13T18:20:03,920 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:03,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T18:20:03,924 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(746): ClusterId : f3a3bf4d-9732-4ab1-9e7c-286aef4c2a8a 2024-11-13T18:20:03,924 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(746): ClusterId : f3a3bf4d-9732-4ab1-9e7c-286aef4c2a8a 2024-11-13T18:20:03,924 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:20:03,924 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:20:03,928 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:20:03,929 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:20:03,929 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:20:03,929 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:20:03,930 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:20:03,930 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:03,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:03,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T18:20:03,936 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:20:03,936 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:20:03,936 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:03,936 DEBUG [RS:1;fda96cf025dc:40711 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b038680, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:20:03,936 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:20:03,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:03,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T18:20:03,937 DEBUG [RS:0;fda96cf025dc:41733 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a5e94e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:20:03,959 DEBUG [RS:0;fda96cf025dc:41733 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fda96cf025dc:41733 2024-11-13T18:20:03,959 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:20:03,959 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:20:03,959 DEBUG [RS:1;fda96cf025dc:40711 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fda96cf025dc:40711 2024-11-13T18:20:03,959 DEBUG [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:20:03,960 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:20:03,960 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:20:03,960 DEBUG [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:20:03,961 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,46067,1731522003442 with port=40711, startcode=1731522003595 2024-11-13T18:20:03,962 DEBUG [RS:1;fda96cf025dc:40711 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:20:03,963 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,46067,1731522003442 with port=41733, startcode=1731522003525 2024-11-13T18:20:03,963 DEBUG [RS:0;fda96cf025dc:41733 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:20:03,968 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:20:03,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:03,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:03,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T18:20:03,970 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41367, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:20:03,970 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46067 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,40711,1731522003595 2024-11-13T18:20:03,970 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46067 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,40711,1731522003595 2024-11-13T18:20:03,972 DEBUG [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15 2024-11-13T18:20:03,973 DEBUG [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T18:20:03,973 DEBUG [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:20:03,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:20:03,976 DEBUG [RS:1;fda96cf025dc:40711 {}] zookeeper.ZKUtil(111): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,40711,1731522003595 2024-11-13T18:20:03,976 WARN [RS:1;fda96cf025dc:40711 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:20:03,976 INFO [RS:1;fda96cf025dc:40711 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:20:03,976 DEBUG [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40711,1731522003595 2024-11-13T18:20:03,983 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36949, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:20:03,984 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,40711,1731522003595] 2024-11-13T18:20:03,984 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46067 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,41733,1731522003525 2024-11-13T18:20:03,985 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46067 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,41733,1731522003525 2024-11-13T18:20:03,985 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:20:03,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:03,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:03,985 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:20:03,986 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740 2024-11-13T18:20:03,986 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740 2024-11-13T18:20:03,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:20:03,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:20:03,987 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:20:03,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:20:03,988 DEBUG [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15 2024-11-13T18:20:03,988 DEBUG [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T18:20:03,988 DEBUG [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:20:03,989 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:20:03,993 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:20:03,994 INFO [RS:1;fda96cf025dc:40711 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:20:03,994 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:03,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:20:03,994 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:20:03,995 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:20:03,995 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:03,995 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,995 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,995 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,996 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,996 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,996 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:20:03,996 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,996 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,996 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,996 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,41733,1731522003525] 2024-11-13T18:20:03,996 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,997 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,997 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:03,997 DEBUG [RS:0;fda96cf025dc:41733 {}] zookeeper.ZKUtil(111): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,41733,1731522003525 2024-11-13T18:20:03,997 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:20:03,997 WARN [RS:0;fda96cf025dc:41733 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:20:03,997 DEBUG [RS:1;fda96cf025dc:40711 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:20:03,997 INFO [RS:0;fda96cf025dc:41733 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:20:03,997 DEBUG [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,41733,1731522003525 2024-11-13T18:20:04,000 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,001 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,001 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,001 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,001 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,001 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,40711,1731522003595-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:20:04,005 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:20:04,005 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71506122, jitterRate=0.06552425026893616}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:20:04,006 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731522003920Initializing all the Stores at 1731522003921 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522003921Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522003921Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522003921Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522003921Cleaning up temporary data from old regions at 1731522003987 (+66 ms)Region opened successfully at 1731522004006 (+19 ms) 2024-11-13T18:20:04,006 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:20:04,006 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:20:04,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:20:04,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:20:04,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:20:04,008 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:20:04,012 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:20:04,012 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:20:04,012 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731522004006Disabling compacts and flushes for region at 1731522004006Disabling writes for close at 1731522004007 (+1 ms)Writing region close event to WAL at 1731522004011 (+4 ms)Closed at 1731522004012 (+1 ms) 2024-11-13T18:20:04,013 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:20:04,013 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T18:20:04,013 INFO [RS:0;fda96cf025dc:41733 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:20:04,013 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T18:20:04,015 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:20:04,016 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T18:20:04,019 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:20:04,019 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:20:04,019 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,020 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:04,020 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:04,020 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:04,020 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:04,020 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:04,020 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:20:04,020 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:04,020 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:04,020 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:04,021 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:04,021 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:04,021 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:04,021 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:20:04,021 DEBUG [RS:0;fda96cf025dc:41733 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:20:04,022 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,022 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,022 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,022 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,022 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,022 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,41733,1731522003525-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:20:04,026 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:20:04,026 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,40711,1731522003595-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,027 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,027 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.Replication(171): fda96cf025dc,40711,1731522003595 started 2024-11-13T18:20:04,042 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,042 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,40711,1731522003595, RpcServer on fda96cf025dc/172.17.0.3:40711, sessionid=0x1003a5f145f0002 2024-11-13T18:20:04,042 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:20:04,042 DEBUG [RS:1;fda96cf025dc:40711 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,40711,1731522003595 2024-11-13T18:20:04,043 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,40711,1731522003595' 2024-11-13T18:20:04,043 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:20:04,043 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:20:04,044 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:20:04,044 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:20:04,044 DEBUG [RS:1;fda96cf025dc:40711 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,40711,1731522003595 2024-11-13T18:20:04,044 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,40711,1731522003595' 2024-11-13T18:20:04,044 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:20:04,045 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:20:04,045 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:20:04,045 DEBUG [RS:1;fda96cf025dc:40711 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:20:04,045 INFO [RS:1;fda96cf025dc:40711 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:20:04,045 INFO [RS:1;fda96cf025dc:40711 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:20:04,046 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,41733,1731522003525-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,046 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,046 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.Replication(171): fda96cf025dc,41733,1731522003525 started 2024-11-13T18:20:04,069 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,070 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,41733,1731522003525, RpcServer on fda96cf025dc/172.17.0.3:41733, sessionid=0x1003a5f145f0001 2024-11-13T18:20:04,070 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:20:04,070 DEBUG [RS:0;fda96cf025dc:41733 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,41733,1731522003525 2024-11-13T18:20:04,070 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,41733,1731522003525' 2024-11-13T18:20:04,070 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:20:04,071 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:20:04,072 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:20:04,072 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:20:04,072 DEBUG [RS:0;fda96cf025dc:41733 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,41733,1731522003525 2024-11-13T18:20:04,072 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,41733,1731522003525' 2024-11-13T18:20:04,072 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:20:04,072 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:20:04,073 DEBUG [RS:0;fda96cf025dc:41733 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:20:04,073 INFO [RS:0;fda96cf025dc:41733 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:20:04,073 INFO [RS:0;fda96cf025dc:41733 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:20:04,147 INFO [RS:1;fda96cf025dc:40711 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:20:04,148 INFO [RS:1;fda96cf025dc:40711 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fda96cf025dc%2C40711%2C1731522003595, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40711,1731522003595, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs, maxLogs=55 2024-11-13T18:20:04,166 WARN [fda96cf025dc:46067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T18:20:04,174 INFO [RS:0;fda96cf025dc:41733 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:20:04,175 INFO [RS:0;fda96cf025dc:41733 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fda96cf025dc%2C41733%2C1731522003525, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,41733,1731522003525, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs, maxLogs=55 2024-11-13T18:20:04,181 INFO [RS:1;fda96cf025dc:40711 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40711,1731522003595/fda96cf025dc%2C40711%2C1731522003595.1731522004150 2024-11-13T18:20:04,183 DEBUG [RS:1;fda96cf025dc:40711 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T18:20:04,209 INFO [RS:0;fda96cf025dc:41733 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,41733,1731522003525/fda96cf025dc%2C41733%2C1731522003525.1731522004177 2024-11-13T18:20:04,211 DEBUG [RS:0;fda96cf025dc:41733 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T18:20:04,417 DEBUG [fda96cf025dc:46067 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T18:20:04,417 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:20:04,418 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:20:04,418 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:20:04,418 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:20:04,418 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:20:04,418 INFO [fda96cf025dc:46067 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:20:04,418 INFO [fda96cf025dc:46067 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:20:04,418 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:20:04,419 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fda96cf025dc,41733,1731522003525 2024-11-13T18:20:04,420 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,41733,1731522003525, state=OPENING 2024-11-13T18:20:04,422 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T18:20:04,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:04,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:04,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:04,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:04,426 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:20:04,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:04,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,41733,1731522003525}] 2024-11-13T18:20:04,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:04,579 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:20:04,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59701, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:20:04,585 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T18:20:04,585 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:20:04,585 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T18:20:04,587 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fda96cf025dc%2C41733%2C1731522003525.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,41733,1731522003525, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs, maxLogs=55 2024-11-13T18:20:04,614 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,41733,1731522003525/fda96cf025dc%2C41733%2C1731522003525.meta.1731522004588.meta 2024-11-13T18:20:04,615 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T18:20:04,615 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:20:04,616 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T18:20:04,616 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T18:20:04,616 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T18:20:04,617 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T18:20:04,617 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:04,617 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T18:20:04,617 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T18:20:04,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T18:20:04,619 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:20:04,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:04,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:04,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T18:20:04,621 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:20:04,621 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:04,621 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:04,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T18:20:04,622 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:20:04,622 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:04,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:04,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T18:20:04,623 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:20:04,623 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:04,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:04,624 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:20:04,625 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740 2024-11-13T18:20:04,625 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740 2024-11-13T18:20:04,625 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:20:04,625 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:20:04,625 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:20:04,626 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:20:04,627 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59092592, jitterRate=-0.11945176124572754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:20:04,627 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T18:20:04,627 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731522004617Writing region info on filesystem at 1731522004617Initializing all the Stores at 1731522004618 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522004618Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522004618Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522004618Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522004618Cleaning up temporary data from old regions at 1731522004625 (+7 ms)Running coprocessor post-open hooks at 1731522004627 (+2 ms)Region opened successfully at 1731522004627 2024-11-13T18:20:04,628 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731522004579 2024-11-13T18:20:04,631 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T18:20:04,631 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T18:20:04,632 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,41733,1731522003525 2024-11-13T18:20:04,633 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,41733,1731522003525, state=OPEN 2024-11-13T18:20:04,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:20:04,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:20:04,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:20:04,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:04,636 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fda96cf025dc,41733,1731522003525 2024-11-13T18:20:04,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:04,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:04,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T18:20:04,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,41733,1731522003525 in 210 msec 2024-11-13T18:20:04,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T18:20:04,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 625 msec 2024-11-13T18:20:04,642 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:20:04,642 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T18:20:04,647 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:20:04,647 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,41733,1731522003525, seqNum=-1] 2024-11-13T18:20:04,648 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:20:04,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37363, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:20:04,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 780 msec 2024-11-13T18:20:04,657 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731522004656, completionTime=-1 2024-11-13T18:20:04,657 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-13T18:20:04,657 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:20:04,659 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T18:20:04,659 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731522064659 2024-11-13T18:20:04,659 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731522124659 2024-11-13T18:20:04,659 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-13T18:20:04,660 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,46067,1731522003442-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,660 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,46067,1731522003442-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,660 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,46067,1731522003442-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,660 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fda96cf025dc:46067, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,660 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,660 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,663 DEBUG [master/fda96cf025dc:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T18:20:04,665 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.011sec 2024-11-13T18:20:04,665 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T18:20:04,665 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T18:20:04,665 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T18:20:04,665 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T18:20:04,665 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T18:20:04,665 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,46067,1731522003442-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:20:04,665 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,46067,1731522003442-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T18:20:04,666 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T18:20:04,666 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T18:20:04,666 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,46067,1731522003442-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:04,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:20:04,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,46067,-1 for getting cluster id 2024-11-13T18:20:04,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:20:04,727 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f3a3bf4d-9732-4ab1-9e7c-286aef4c2a8a' 2024-11-13T18:20:04,728 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:20:04,728 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f3a3bf4d-9732-4ab1-9e7c-286aef4c2a8a" 2024-11-13T18:20:04,728 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1072fd1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:20:04,728 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,46067,-1] 2024-11-13T18:20:04,729 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:20:04,729 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:04,731 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:20:04,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:20:04,733 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:20:04,734 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,41733,1731522003525, seqNum=-1] 2024-11-13T18:20:04,735 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:20:04,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:20:04,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fda96cf025dc,46067,1731522003442 2024-11-13T18:20:04,739 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:20:04,740 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is fda96cf025dc,46067,1731522003442 2024-11-13T18:20:04,740 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10a5330d 2024-11-13T18:20:04,740 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:20:04,742 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:20:04,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46067 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnGracefulStop1731522004738', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:20:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46067 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731522004738 2024-11-13T18:20:04,745 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731522004738 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:20:04,745 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:04,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46067 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731522004738" procId is: 4 2024-11-13T18:20:04,746 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731522004738 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:20:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:20:04,760 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731522004738-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8d531c884d866a2490dfd2337a4db1a8, NAME => 'testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731522004738', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15 2024-11-13T18:20:04,768 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731522004738-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:04,768 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731522004738-pool-0 {}] regionserver.HRegion(1722): Closing 8d531c884d866a2490dfd2337a4db1a8, disabling compactions & flushes 2024-11-13T18:20:04,768 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731522004738-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:04,768 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731522004738-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:04,768 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731522004738-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. after waiting 0 ms 2024-11-13T18:20:04,768 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731522004738-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:04,768 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731522004738-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:04,768 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731522004738-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8d531c884d866a2490dfd2337a4db1a8: Waiting for close lock at 1731522004768Disabling compacts and flushes for region at 1731522004768Disabling writes for close at 1731522004768Writing region close event to WAL at 1731522004768Closed at 1731522004768 2024-11-13T18:20:04,769 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731522004738 execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:20:04,769 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731522004769"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731522004769"}]},"ts":"1731522004769"} 2024-11-13T18:20:04,772 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:20:04,773 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731522004738 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:20:04,773 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731522004738","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522004773"}]},"ts":"1731522004773"} 2024-11-13T18:20:04,775 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731522004738, state=ENABLING in hbase:meta 2024-11-13T18:20:04,776 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:20:04,776 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:20:04,776 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:20:04,776 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:20:04,776 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:20:04,776 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:20:04,776 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:20:04,776 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:20:04,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731522004738, region=8d531c884d866a2490dfd2337a4db1a8, ASSIGN}] 2024-11-13T18:20:04,777 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731522004738, region=8d531c884d866a2490dfd2337a4db1a8, ASSIGN 2024-11-13T18:20:04,778 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731522004738, region=8d531c884d866a2490dfd2337a4db1a8, ASSIGN; state=OFFLINE, location=fda96cf025dc,41733,1731522003525; forceNewPlan=false, retain=false 2024-11-13T18:20:04,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:20:04,929 INFO [fda96cf025dc:46067 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T18:20:04,929 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8d531c884d866a2490dfd2337a4db1a8, regionState=OPENING, regionLocation=fda96cf025dc,41733,1731522003525 2024-11-13T18:20:04,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731522004738, region=8d531c884d866a2490dfd2337a4db1a8, ASSIGN because future has completed 2024-11-13T18:20:04,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d531c884d866a2490dfd2337a4db1a8, server=fda96cf025dc,41733,1731522003525}] 2024-11-13T18:20:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:20:05,090 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:05,091 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d531c884d866a2490dfd2337a4db1a8, NAME => 'testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:20:05,091 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731522004738 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:05,091 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:05,092 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:05,092 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:05,095 INFO [StoreOpener-8d531c884d866a2490dfd2337a4db1a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:05,101 INFO [StoreOpener-8d531c884d866a2490dfd2337a4db1a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d531c884d866a2490dfd2337a4db1a8 columnFamilyName family 2024-11-13T18:20:05,101 DEBUG [StoreOpener-8d531c884d866a2490dfd2337a4db1a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:05,102 INFO [StoreOpener-8d531c884d866a2490dfd2337a4db1a8-1 {}] regionserver.HStore(327): Store=8d531c884d866a2490dfd2337a4db1a8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:20:05,102 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:05,102 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:05,103 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:05,103 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:05,103 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:05,104 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:05,115 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:20:05,116 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8d531c884d866a2490dfd2337a4db1a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65676642, jitterRate=-0.021341770887374878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:20:05,116 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:05,117 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8d531c884d866a2490dfd2337a4db1a8: Running coprocessor pre-open hook at 1731522005092Writing region info on filesystem at 1731522005092Initializing all the Stores at 1731522005092Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522005092Cleaning up temporary data from old regions at 1731522005103 (+11 ms)Running coprocessor post-open hooks at 1731522005116 (+13 ms)Region opened successfully at 1731522005117 (+1 ms) 2024-11-13T18:20:05,124 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8., pid=6, masterSystemTime=1731522005087 2024-11-13T18:20:05,127 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:05,127 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:05,128 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8d531c884d866a2490dfd2337a4db1a8, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,41733,1731522003525 2024-11-13T18:20:05,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d531c884d866a2490dfd2337a4db1a8, server=fda96cf025dc,41733,1731522003525 because future has completed 2024-11-13T18:20:05,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T18:20:05,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d531c884d866a2490dfd2337a4db1a8, server=fda96cf025dc,41733,1731522003525 in 197 msec 2024-11-13T18:20:05,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T18:20:05,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731522004738, region=8d531c884d866a2490dfd2337a4db1a8, ASSIGN in 359 msec 2024-11-13T18:20:05,138 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731522004738 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:20:05,139 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731522004738","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522005138"}]},"ts":"1731522005138"} 2024-11-13T18:20:05,141 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731522004738, state=ENABLED in hbase:meta 2024-11-13T18:20:05,142 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731522004738 execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:20:05,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731522004738 in 400 msec 2024-11-13T18:20:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:20:05,372 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731522004738 completed 2024-11-13T18:20:05,372 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731522004738 get assigned. Timeout = 60000ms 2024-11-13T18:20:05,372 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:20:05,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731522004738 assigned to meta. Checking AM states. 2024-11-13T18:20:05,377 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:20:05,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731522004738 assigned. 2024-11-13T18:20:05,381 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731522004738', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8., hostname=fda96cf025dc,41733,1731522003525, seqNum=2] 2024-11-13T18:20:05,387 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8d531c884d866a2490dfd2337a4db1a8 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-13T18:20:05,401 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9396570bcfce4c7881bcb3d9b78b7678 2024-11-13T18:20:05,419 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/.tmp/family/9396570bcfce4c7881bcb3d9b78b7678 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-13T18:20:05,420 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/.tmp/family/9396570bcfce4c7881bcb3d9b78b7678 2024-11-13T18:20:05,423 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/.tmp/family/9396570bcfce4c7881bcb3d9b78b7678 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/family/9396570bcfce4c7881bcb3d9b78b7678 2024-11-13T18:20:05,424 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/family/9396570bcfce4c7881bcb3d9b78b7678, delay=919 ms 2024-11-13T18:20:05,426 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/family/9396570bcfce4c7881bcb3d9b78b7678, entries=2, sequenceid=6, filesize=4.9 K 2024-11-13T18:20:05,426 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 8d531c884d866a2490dfd2337a4db1a8 in 39ms, sequenceid=6, compaction requested=false 2024-11-13T18:20:05,426 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731522004738' 2024-11-13T18:20:05,427 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8d531c884d866a2490dfd2337a4db1a8: 2024-11-13T18:20:05,566 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:20:06,344 DEBUG [hfile-prefetch-1731522002847 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9396570bcfce4c7881bcb3d9b78b7678_0. 2024-11-13T18:20:06,345 DEBUG [hfile-prefetch-1731522002847 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9396570bcfce4c7881bcb3d9b78b7678_117. 2024-11-13T18:20:06,345 DEBUG [hfile-prefetch-1731522002847 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/family/9396570bcfce4c7881bcb3d9b78b7678, with total blocks 2, and data blocks 1 2024-11-13T18:20:06,345 DEBUG [hfile-prefetch-1731522002847 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-13T18:20:06,346 DEBUG [hfile-prefetch-1731522002847 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/family/9396570bcfce4c7881bcb3d9b78b7678, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-13T18:20:06,346 DEBUG [hfile-prefetch-1731522002847 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/family/9396570bcfce4c7881bcb3d9b78b7678. Will sleep 100ms and try the verification again. 2024-11-13T18:20:06,413 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-13T18:20:06,413 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-13T18:20:06,414 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T18:20:06,414 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T18:20:06,418 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T18:20:06,418 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731522006413 2024-11-13T18:20:06,418 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-13T18:20:06,428 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;fda96cf025dc:41733,5,FailOnTimeoutGroup] 2024-11-13T18:20:06,428 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,41733,1731522003525' ***** 2024-11-13T18:20:06,428 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-13T18:20:06,428 INFO [RS:0;fda96cf025dc:41733 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T18:20:06,428 INFO [RS:0;fda96cf025dc:41733 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.persistence 2024-11-13T18:20:06,428 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T18:20:06,429 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T18:20:06,429 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T18:20:06,429 INFO [RS:0;fda96cf025dc:41733 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T18:20:06,429 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T18:20:06,439 DEBUG [RS:0;fda96cf025dc:41733 {}] bucket.BucketCache(1488): Thread RS:0;fda96cf025dc:41733 started persisting bucket cache to file 2024-11-13T18:20:06,439 DEBUG [RS:0;fda96cf025dc:41733 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T18:20:06,439 DEBUG [RS:0;fda96cf025dc:41733 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T18:20:06,446 DEBUG [hfile-prefetch-1731522002847 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/family/9396570bcfce4c7881bcb3d9b78b7678, with total blocks 2, and data blocks 1 2024-11-13T18:20:06,446 DEBUG [hfile-prefetch-1731522002847 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-13T18:20:06,446 DEBUG [hfile-prefetch-1731522002847 {}] bucket.BucketCache(2294): found block for file 9396570bcfce4c7881bcb3d9b78b7678 in the backing map. Acquiring read lock for offset 0 2024-11-13T18:20:06,446 DEBUG [hfile-prefetch-1731522002847 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/family/9396570bcfce4c7881bcb3d9b78b7678 has now been fully cached. 2024-11-13T18:20:06,446 DEBUG [hfile-prefetch-1731522002847 {}] hfile.PrefetchExecutor(121): Prefetch completed for 9396570bcfce4c7881bcb3d9b78b7678 2024-11-13T18:20:06,449 DEBUG [RS:0;fda96cf025dc:41733 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-13T18:20:06,449 DEBUG [RS:0;fda96cf025dc:41733 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731522006439 2024-11-13T18:20:06,449 DEBUG [RS:0;fda96cf025dc:41733 {}] bucket.BucketCache(1510): Thread RS:0;fda96cf025dc:41733 finished persisting bucket cache to file, renaming 2024-11-13T18:20:06,449 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:20:06,449 INFO [RS:0;fda96cf025dc:41733 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:20:06,449 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:20:06,449 INFO [RS:0;fda96cf025dc:41733 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:20:06,449 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(3091): Received CLOSE for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:06,450 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(959): stopping server fda96cf025dc,41733,1731522003525 2024-11-13T18:20:06,450 INFO [RS:0;fda96cf025dc:41733 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:20:06,450 INFO [RS:0;fda96cf025dc:41733 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fda96cf025dc:41733. 2024-11-13T18:20:06,450 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T18:20:06,450 DEBUG [RS:0;fda96cf025dc:41733 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:20:06,450 DEBUG [RS:0;fda96cf025dc:41733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:06,450 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8d531c884d866a2490dfd2337a4db1a8, disabling compactions & flushes 2024-11-13T18:20:06,450 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:20:06,450 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:06,450 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:20:06,450 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:20:06,450 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:06,450 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T18:20:06,450 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. after waiting 0 ms 2024-11-13T18:20:06,451 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:06,452 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T18:20:06,452 DEBUG [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 8d531c884d866a2490dfd2337a4db1a8=testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8.} 2024-11-13T18:20:06,453 DEBUG [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:06,453 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T18:20:06,453 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:20:06,453 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:20:06,453 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:20:06,453 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:20:06,453 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:20:06,453 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-13T18:20:06,470 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-13T18:20:06,470 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:06,470 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8d531c884d866a2490dfd2337a4db1a8: Waiting for close lock at 1731522006450Running coprocessor pre-close hooks at 1731522006450Disabling compacts and flushes for region at 1731522006450Disabling writes for close at 1731522006451 (+1 ms)Writing region close event to WAL at 1731522006462 (+11 ms)Running coprocessor post-close hooks at 1731522006470 (+8 ms)Closed at 1731522006470 2024-11-13T18:20:06,471 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:06,471 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6f609e53064242bcb9e96b20f3a02bc0 2024-11-13T18:20:06,491 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/info/6f609e53064242bcb9e96b20f3a02bc0 is 209, key is testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8./info:regioninfo/1731522005128/Put/seqid=0 2024-11-13T18:20:06,492 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/info/6f609e53064242bcb9e96b20f3a02bc0 2024-11-13T18:20:06,503 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5584919e553748a2a6a9ac6f9b05bc08 2024-11-13T18:20:06,520 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/ns/5584919e553748a2a6a9ac6f9b05bc08 is 43, key is default/ns:d/1731522004651/Put/seqid=0 2024-11-13T18:20:06,521 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/ns/5584919e553748a2a6a9ac6f9b05bc08 2024-11-13T18:20:06,528 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:20:06,531 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on df7640473dec46e3b9d667a9f8734234 2024-11-13T18:20:06,551 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/table/df7640473dec46e3b9d667a9f8734234 is 80, key is testBlockEvictionOnGracefulStop1731522004738/table:state/1731522005138/Put/seqid=0 2024-11-13T18:20:06,552 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/table/df7640473dec46e3b9d667a9f8734234 2024-11-13T18:20:06,556 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/info/6f609e53064242bcb9e96b20f3a02bc0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/6f609e53064242bcb9e96b20f3a02bc0 2024-11-13T18:20:06,557 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/6f609e53064242bcb9e96b20f3a02bc0, delay=947 ms 2024-11-13T18:20:06,558 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/6f609e53064242bcb9e96b20f3a02bc0, entries=10, sequenceid=11, filesize=7.0 K 2024-11-13T18:20:06,559 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/ns/5584919e553748a2a6a9ac6f9b05bc08 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/ns/5584919e553748a2a6a9ac6f9b05bc08 2024-11-13T18:20:06,560 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/ns/5584919e553748a2a6a9ac6f9b05bc08, delay=939 ms 2024-11-13T18:20:06,561 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/ns/5584919e553748a2a6a9ac6f9b05bc08, entries=2, sequenceid=11, filesize=5.0 K 2024-11-13T18:20:06,561 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/table/df7640473dec46e3b9d667a9f8734234 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/table/df7640473dec46e3b9d667a9f8734234 2024-11-13T18:20:06,562 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/table/df7640473dec46e3b9d667a9f8734234, delay=903 ms 2024-11-13T18:20:06,563 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/table/df7640473dec46e3b9d667a9f8734234, entries=2, sequenceid=11, filesize=5.3 K 2024-11-13T18:20:06,564 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=11, compaction requested=false 2024-11-13T18:20:06,565 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/6f609e53064242bcb9e96b20f3a02bc0 2024-11-13T18:20:06,565 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/ns/5584919e553748a2a6a9ac6f9b05bc08 2024-11-13T18:20:06,566 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/table/df7640473dec46e3b9d667a9f8734234 2024-11-13T18:20:06,570 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-13T18:20:06,571 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:20:06,571 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:20:06,571 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731522006452Running coprocessor pre-close hooks at 1731522006453 (+1 ms)Disabling compacts and flushes for region at 1731522006453Disabling writes for close at 1731522006453Obtaining lock to block concurrent updates at 1731522006453Preparing flush snapshotting stores in 1588230740 at 1731522006453Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731522006454 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731522006454Flushing 1588230740/info: creating writer at 1731522006454Flushing 1588230740/info: appending metadata at 1731522006491 (+37 ms)Flushing 1588230740/info: closing flushed file at 1731522006491Flushing 1588230740/ns: creating writer at 1731522006495 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731522006519 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1731522006519Flushing 1588230740/table: creating writer at 1731522006524 (+5 ms)Flushing 1588230740/table: appending metadata at 1731522006550 (+26 ms)Flushing 1588230740/table: closing flushed file at 1731522006550Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a9d3be0: reopening flushed file at 1731522006555 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f80d4f5: reopening flushed file at 1731522006558 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72d4bac4: reopening flushed file at 1731522006561 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=11, compaction requested=false at 1731522006564 (+3 ms)Writing region close event to WAL at 1731522006566 (+2 ms)Running coprocessor post-close hooks at 1731522006571 (+5 ms)Closed at 1731522006571 2024-11-13T18:20:06,571 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T18:20:06,653 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(976): stopping server fda96cf025dc,41733,1731522003525; all regions closed. 2024-11-13T18:20:06,656 DEBUG [RS:0;fda96cf025dc:41733 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs 2024-11-13T18:20:06,656 INFO [RS:0;fda96cf025dc:41733 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C41733%2C1731522003525.meta:.meta(num 1731522004588) 2024-11-13T18:20:06,661 DEBUG [RS:0;fda96cf025dc:41733 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs 2024-11-13T18:20:06,661 INFO [RS:0;fda96cf025dc:41733 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C41733%2C1731522003525:(num 1731522004177) 2024-11-13T18:20:06,661 DEBUG [RS:0;fda96cf025dc:41733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:06,661 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:20:06,661 INFO [RS:0;fda96cf025dc:41733 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:20:06,662 INFO [RS:0;fda96cf025dc:41733 {}] hbase.ChoreService(370): Chore service for: regionserver/fda96cf025dc:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T18:20:06,662 INFO [RS:0;fda96cf025dc:41733 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:20:06,662 INFO [RS:0;fda96cf025dc:41733 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41733 2024-11-13T18:20:06,663 INFO [regionserver/fda96cf025dc:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:20:06,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,41733,1731522003525 2024-11-13T18:20:06,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:20:06,668 INFO [RS:0;fda96cf025dc:41733 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:20:06,669 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,41733,1731522003525] 2024-11-13T18:20:06,671 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,41733,1731522003525 already deleted, retry=false 2024-11-13T18:20:06,672 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of fda96cf025dc,41733,1731522003525 on fda96cf025dc,46067,1731522003442 2024-11-13T18:20:06,677 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fda96cf025dc,41733,1731522003525, splitWal=true, meta=true 2024-11-13T18:20:06,680 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure fda96cf025dc,41733,1731522003525, splitWal=true, meta=true 2024-11-13T18:20:06,681 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for fda96cf025dc,41733,1731522003525 (carryingMeta=true) fda96cf025dc,41733,1731522003525/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@70c8feae[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T18:20:06,682 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,41733,1731522003525, splitWal=true, meta=true, isMeta: true 2024-11-13T18:20:06,682 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,41733,1731522003525-splitting 2024-11-13T18:20:06,683 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,41733,1731522003525-splitting dir is empty, no logs to split. 2024-11-13T18:20:06,683 INFO [PEWorker-2 {}] master.SplitWALManager(105): fda96cf025dc,41733,1731522003525 WAL count=0, meta=true 2024-11-13T18:20:06,686 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,41733,1731522003525-splitting dir is empty, no logs to split. 2024-11-13T18:20:06,686 INFO [PEWorker-2 {}] master.SplitWALManager(105): fda96cf025dc,41733,1731522003525 WAL count=0, meta=true 2024-11-13T18:20:06,686 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if fda96cf025dc,41733,1731522003525 WAL splitting is done? wals=0, meta=true 2024-11-13T18:20:06,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T18:20:06,689 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:20:06,690 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:20:06,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:06,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41733-0x1003a5f145f0001, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:06,771 INFO [RS:0;fda96cf025dc:41733 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:20:06,771 INFO [RS:0;fda96cf025dc:41733 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,41733,1731522003525; zookeeper connection closed. 2024-11-13T18:20:06,771 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6f13028e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6f13028e 2024-11-13T18:20:06,840 DEBUG [fda96cf025dc:46067 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T18:20:06,841 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fda96cf025dc,40711,1731522003595 2024-11-13T18:20:06,842 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,40711,1731522003595, state=OPENING 2024-11-13T18:20:06,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:20:06,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:20:06,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:06,845 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:20:06,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:06,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,40711,1731522003595}] 2024-11-13T18:20:06,949 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:20:06,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:06,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:06,950 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:20:06,950 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:20:06,950 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:20:06,950 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:20:06,950 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:20:06,951 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36045 2024-11-13T18:20:06,952 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36045 connecting to ZooKeeper ensemble=127.0.0.1:56021 2024-11-13T18:20:06,953 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:06,953 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:06,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360450x0, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:20:06,957 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:360450x0, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-13T18:20:06,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36045-0x1003a5f145f0003 connected 2024-11-13T18:20:06,957 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-13T18:20:06,958 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:20:06,961 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.cache 2024-11-13T18:20:06,961 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T18:20:06,961 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T18:20:06,961 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T18:20:06,961 INFO [Thread-207 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T18:20:06,962 INFO [Thread-207 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-13T18:20:06,964 INFO [Thread-207 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-13T18:20:06,964 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T18:20:06,965 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:20:06,965 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36045-0x1003a5f145f0003, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T18:20:06,968 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36045-0x1003a5f145f0003, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:20:06,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36045 2024-11-13T18:20:06,969 DEBUG [Thread-207 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-13T18:20:06,969 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T18:20:06,970 INFO [Thread-207 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-13T18:20:06,970 INFO [Thread-207 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.persistence is complete. 2024-11-13T18:20:06,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36045 2024-11-13T18:20:06,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36045 2024-11-13T18:20:06,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36045 2024-11-13T18:20:06,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36045 2024-11-13T18:20:06,978 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer(746): ClusterId : f3a3bf4d-9732-4ab1-9e7c-286aef4c2a8a 2024-11-13T18:20:06,978 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:20:06,981 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:20:06,981 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:20:06,984 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:20:06,984 DEBUG [RS:2;fda96cf025dc:36045 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b781dbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:20:07,001 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:20:07,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35257, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:20:07,006 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T18:20:07,006 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:20:07,006 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T18:20:07,007 DEBUG [RS:2;fda96cf025dc:36045 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;fda96cf025dc:36045 2024-11-13T18:20:07,007 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:20:07,007 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:20:07,007 DEBUG [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:20:07,007 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fda96cf025dc%2C40711%2C1731522003595.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40711,1731522003595, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs, maxLogs=55 2024-11-13T18:20:07,009 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,46067,1731522003442 with port=36045, startcode=1731522006949 2024-11-13T18:20:07,009 DEBUG [RS:2;fda96cf025dc:36045 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:20:07,012 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46055, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:20:07,012 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46067 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,36045,1731522006949 2024-11-13T18:20:07,012 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46067 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,36045,1731522006949 2024-11-13T18:20:07,015 DEBUG [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15 2024-11-13T18:20:07,015 DEBUG [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T18:20:07,015 DEBUG [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:20:07,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:20:07,019 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,36045,1731522006949] 2024-11-13T18:20:07,019 DEBUG [RS:2;fda96cf025dc:36045 {}] zookeeper.ZKUtil(111): regionserver:36045-0x1003a5f145f0003, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,36045,1731522006949 2024-11-13T18:20:07,019 WARN [RS:2;fda96cf025dc:36045 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:20:07,019 INFO [RS:2;fda96cf025dc:36045 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:20:07,019 DEBUG [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,36045,1731522006949 2024-11-13T18:20:07,020 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T18:20:07,023 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T18:20:07,026 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:20:07,030 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:20:07,031 INFO [RS:2;fda96cf025dc:36045 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:20:07,031 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:07,031 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:20:07,032 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:20:07,032 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:07,032 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:07,032 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:07,032 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:07,032 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:07,032 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:07,032 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:20:07,032 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:07,033 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:07,033 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:07,033 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:07,033 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:07,033 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:20:07,033 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:20:07,033 DEBUG [RS:2;fda96cf025dc:36045 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:20:07,040 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:07,041 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:07,041 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:07,041 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:07,041 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:07,041 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,36045,1731522006949-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:20:07,045 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,40711,1731522003595/fda96cf025dc%2C40711%2C1731522003595.meta.1731522007008.meta 2024-11-13T18:20:07,045 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T18:20:07,045 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:20:07,046 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T18:20:07,046 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T18:20:07,046 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T18:20:07,046 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T18:20:07,046 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:07,046 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T18:20:07,047 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T18:20:07,048 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T18:20:07,048 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:20:07,049 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:07,051 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/6f609e53064242bcb9e96b20f3a02bc0, delay=947 ms 2024-11-13T18:20:07,052 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/6f609e53064242bcb9e96b20f3a02bc0 2024-11-13T18:20:07,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:07,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T18:20:07,053 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:20:07,053 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:07,058 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/ns/5584919e553748a2a6a9ac6f9b05bc08, delay=975 ms 2024-11-13T18:20:07,063 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/ns/5584919e553748a2a6a9ac6f9b05bc08 2024-11-13T18:20:07,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:07,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T18:20:07,064 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:20:07,064 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:07,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:07,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T18:20:07,065 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:20:07,065 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:07,067 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/table/df7640473dec46e3b9d667a9f8734234, delay=980 ms 2024-11-13T18:20:07,069 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/table/df7640473dec46e3b9d667a9f8734234 2024-11-13T18:20:07,069 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:20:07,069 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:20:07,069 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,36045,1731522006949-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:07,069 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:20:07,070 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:07,070 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.Replication(171): fda96cf025dc,36045,1731522006949 started 2024-11-13T18:20:07,070 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740 2024-11-13T18:20:07,071 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740 2024-11-13T18:20:07,071 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:20:07,071 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:20:07,072 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:20:07,073 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:20:07,074 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62504398, jitterRate=-0.06861189007759094}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:20:07,074 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T18:20:07,075 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731522007047Writing region info on filesystem at 1731522007047Initializing all the Stores at 1731522007047Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522007047Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522007047Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522007047Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522007047Cleaning up temporary data from old regions at 1731522007071 (+24 ms)Running coprocessor post-open hooks at 1731522007074 (+3 ms)Region opened successfully at 1731522007075 (+1 ms) 2024-11-13T18:20:07,076 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731522007001 2024-11-13T18:20:07,080 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T18:20:07,080 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T18:20:07,081 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=fda96cf025dc,40711,1731522003595 2024-11-13T18:20:07,082 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,40711,1731522003595, state=OPEN 2024-11-13T18:20:07,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36045-0x1003a5f145f0003, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:20:07,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:20:07,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:20:07,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:07,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:07,085 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:20:07,085 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fda96cf025dc,40711,1731522003595 2024-11-13T18:20:07,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-13T18:20:07,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,40711,1731522003595 in 240 msec 2024-11-13T18:20:07,090 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:20:07,090 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,36045,1731522006949, RpcServer on fda96cf025dc/172.17.0.3:36045, sessionid=0x1003a5f145f0003 2024-11-13T18:20:07,090 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:20:07,090 DEBUG [RS:2;fda96cf025dc:36045 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,36045,1731522006949 2024-11-13T18:20:07,090 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,36045,1731522006949' 2024-11-13T18:20:07,090 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:20:07,091 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:20:07,091 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): fda96cf025dc,41733,1731522003525 had 2 regions 2024-11-13T18:20:07,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T18:20:07,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 401 msec 2024-11-13T18:20:07,092 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:20:07,092 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,41733,1731522003525, splitWal=true, meta=true, isMeta: false 2024-11-13T18:20:07,092 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:20:07,092 DEBUG [RS:2;fda96cf025dc:36045 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,36045,1731522006949 2024-11-13T18:20:07,092 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,36045,1731522006949' 2024-11-13T18:20:07,092 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:20:07,093 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,41733,1731522003525-splitting dir is empty, no logs to split. 2024-11-13T18:20:07,093 INFO [PEWorker-3 {}] master.SplitWALManager(105): fda96cf025dc,41733,1731522003525 WAL count=0, meta=false 2024-11-13T18:20:07,093 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:20:07,094 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,41733,1731522003525-splitting dir is empty, no logs to split. 2024-11-13T18:20:07,094 INFO [PEWorker-3 {}] master.SplitWALManager(105): fda96cf025dc,41733,1731522003525 WAL count=0, meta=false 2024-11-13T18:20:07,094 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if fda96cf025dc,41733,1731522003525 WAL splitting is done? wals=0, meta=false 2024-11-13T18:20:07,095 DEBUG [RS:2;fda96cf025dc:36045 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:20:07,095 INFO [RS:2;fda96cf025dc:36045 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:20:07,095 INFO [RS:2;fda96cf025dc:36045 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:20:07,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731522004738, region=8d531c884d866a2490dfd2337a4db1a8, ASSIGN}] 2024-11-13T18:20:07,097 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731522004738, region=8d531c884d866a2490dfd2337a4db1a8, ASSIGN 2024-11-13T18:20:07,098 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731522004738, region=8d531c884d866a2490dfd2337a4db1a8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:20:07,196 INFO [RS:2;fda96cf025dc:36045 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:20:07,197 INFO [RS:2;fda96cf025dc:36045 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fda96cf025dc%2C36045%2C1731522006949, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,36045,1731522006949, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs, maxLogs=55 2024-11-13T18:20:07,221 INFO [RS:2;fda96cf025dc:36045 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/WALs/fda96cf025dc,36045,1731522006949/fda96cf025dc%2C36045%2C1731522006949.1731522007197 2024-11-13T18:20:07,224 DEBUG [RS:2;fda96cf025dc:36045 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T18:20:07,249 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:20:07,249 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:20:07,249 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:20:07,249 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:20:07,249 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:20:07,249 INFO [fda96cf025dc:46067 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:20:07,249 INFO [fda96cf025dc:46067 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:20:07,249 DEBUG [fda96cf025dc:46067 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:20:07,250 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8d531c884d866a2490dfd2337a4db1a8, regionState=OPENING, regionLocation=fda96cf025dc,40711,1731522003595 2024-11-13T18:20:07,251 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:41733 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:41733 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:20:07,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,41733,1731522003525, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,41733,1731522003525, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:41733 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:41733 2024-11-13T18:20:07,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,41733,1731522003525, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:20:07,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,41733,1731522003525, seqNum=-1 from cache 2024-11-13T18:20:07,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:41733 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:41733 2024-11-13T18:20:07,360 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:20:07,361 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,40711,1731522003595, seqNum=-1] 2024-11-13T18:20:07,361 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:20:07,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45333, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:20:07,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731522004738, region=8d531c884d866a2490dfd2337a4db1a8, ASSIGN because future has completed 2024-11-13T18:20:07,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d531c884d866a2490dfd2337a4db1a8, server=fda96cf025dc,40711,1731522003595}] 2024-11-13T18:20:07,526 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:07,526 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d531c884d866a2490dfd2337a4db1a8, NAME => 'testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:20:07,527 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731522004738 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,527 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:20:07,527 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,527 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,528 INFO [StoreOpener-8d531c884d866a2490dfd2337a4db1a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,529 INFO [StoreOpener-8d531c884d866a2490dfd2337a4db1a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d531c884d866a2490dfd2337a4db1a8 columnFamilyName family 2024-11-13T18:20:07,529 DEBUG [StoreOpener-8d531c884d866a2490dfd2337a4db1a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:20:07,531 DEBUG [StoreFileOpener-8d531c884d866a2490dfd2337a4db1a8-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/family/9396570bcfce4c7881bcb3d9b78b7678, delay=912 ms 2024-11-13T18:20:07,532 DEBUG [StoreOpener-8d531c884d866a2490dfd2337a4db1a8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/family/9396570bcfce4c7881bcb3d9b78b7678 2024-11-13T18:20:07,532 INFO [StoreOpener-8d531c884d866a2490dfd2337a4db1a8-1 {}] regionserver.HStore(327): Store=8d531c884d866a2490dfd2337a4db1a8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:20:07,532 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,533 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,534 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,534 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,534 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,536 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,536 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 8d531c884d866a2490dfd2337a4db1a8; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73245013, jitterRate=0.0914357453584671}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:20:07,536 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,552 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 8d531c884d866a2490dfd2337a4db1a8: Running coprocessor pre-open hook at 1731522007527Writing region info on filesystem at 1731522007527Initializing all the Stores at 1731522007528 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522007528Cleaning up temporary data from old regions at 1731522007534 (+6 ms)Running coprocessor post-open hooks at 1731522007551 (+17 ms)Region opened successfully at 1731522007552 (+1 ms) 2024-11-13T18:20:07,557 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8., pid=11, masterSystemTime=1731522007523 2024-11-13T18:20:07,561 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:07,562 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:07,562 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8d531c884d866a2490dfd2337a4db1a8, regionState=OPEN, openSeqNum=10, regionLocation=fda96cf025dc,40711,1731522003595 2024-11-13T18:20:07,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d531c884d866a2490dfd2337a4db1a8, server=fda96cf025dc,40711,1731522003595 because future has completed 2024-11-13T18:20:07,575 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46067 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fda96cf025dc,40711,1731522003595, table=testBlockEvictionOnGracefulStop1731522004738, region=8d531c884d866a2490dfd2337a4db1a8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:20:07,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T18:20:07,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d531c884d866a2490dfd2337a4db1a8, server=fda96cf025dc,40711,1731522003595 in 211 msec 2024-11-13T18:20:07,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-13T18:20:07,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731522004738, region=8d531c884d866a2490dfd2337a4db1a8, ASSIGN in 483 msec 2024-11-13T18:20:07,585 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server fda96cf025dc,41733,1731522003525 after splitting done 2024-11-13T18:20:07,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure fda96cf025dc,41733,1731522003525, splitWal=true, meta=true in 912 msec 2024-11-13T18:20:07,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T18:20:07,590 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:20:07,591 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:20:07,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:07,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:07,591 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T18:20:07,591 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1838120439, stopped=false 2024-11-13T18:20:07,591 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fda96cf025dc,46067,1731522003442 2024-11-13T18:20:07,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:20:07,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36045-0x1003a5f145f0003, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:20:07,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36045-0x1003a5f145f0003, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:07,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:20:07,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:07,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:20:07,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:07,597 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:20:07,598 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:20:07,598 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:20:07,598 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36045-0x1003a5f145f0003, quorum=127.0.0.1:56021, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:20:07,599 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:20:07,599 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:20:07,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:07,600 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,40711,1731522003595' ***** 2024-11-13T18:20:07,600 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:20:07,600 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,36045,1731522006949' ***** 2024-11-13T18:20:07,600 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:20:07,600 INFO [RS:2;fda96cf025dc:36045 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T18:20:07,600 INFO [RS:1;fda96cf025dc:40711 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T18:20:07,600 INFO [RS:1;fda96cf025dc:40711 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.persistence 2024-11-13T18:20:07,600 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T18:20:07,600 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T18:20:07,600 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T18:20:07,601 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T18:20:07,600 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T18:20:07,601 INFO [RS:1;fda96cf025dc:40711 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T18:20:07,601 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T18:20:07,601 INFO [RS:2;fda96cf025dc:36045 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/19622564-d1f4-1879-8371-43b9632b515b/bucket.persistence 2024-11-13T18:20:07,601 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T18:20:07,601 INFO [RS:2;fda96cf025dc:36045 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T18:20:07,601 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T18:20:07,608 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:20:07,611 DEBUG [RS:1;fda96cf025dc:40711 {}] bucket.BucketCache(1488): Thread RS:1;fda96cf025dc:40711 started persisting bucket cache to file 2024-11-13T18:20:07,611 DEBUG [RS:1;fda96cf025dc:40711 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T18:20:07,611 DEBUG [RS:1;fda96cf025dc:40711 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T18:20:07,611 DEBUG [RS:2;fda96cf025dc:36045 {}] bucket.BucketCache(1488): Thread RS:2;fda96cf025dc:36045 started persisting bucket cache to file 2024-11-13T18:20:07,612 DEBUG [RS:2;fda96cf025dc:36045 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T18:20:07,612 DEBUG [RS:2;fda96cf025dc:36045 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T18:20:07,614 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:20:07,626 DEBUG [RS:1;fda96cf025dc:40711 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T18:20:07,626 DEBUG [RS:1;fda96cf025dc:40711 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731522007611 2024-11-13T18:20:07,627 DEBUG [RS:1;fda96cf025dc:40711 {}] bucket.BucketCache(1510): Thread RS:1;fda96cf025dc:40711 finished persisting bucket cache to file, renaming 2024-11-13T18:20:07,627 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:20:07,627 INFO [RS:1;fda96cf025dc:40711 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:20:07,627 INFO [RS:1;fda96cf025dc:40711 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:20:07,627 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(3091): Received CLOSE for 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,628 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(959): stopping server fda96cf025dc,40711,1731522003595 2024-11-13T18:20:07,628 INFO [RS:1;fda96cf025dc:40711 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:20:07,629 INFO [RS:1;fda96cf025dc:40711 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fda96cf025dc:40711. 2024-11-13T18:20:07,628 WARN [Thread-217 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-13T18:20:07,629 DEBUG [RS:1;fda96cf025dc:40711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:20:07,629 DEBUG [RS:1;fda96cf025dc:40711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:07,629 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:20:07,629 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:20:07,629 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:20:07,629 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T18:20:07,630 DEBUG [RS:2;fda96cf025dc:36045 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-13T18:20:07,630 DEBUG [RS:2;fda96cf025dc:36045 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731522007611 2024-11-13T18:20:07,630 DEBUG [RS:2;fda96cf025dc:36045 {}] bucket.BucketCache(1510): Thread RS:2;fda96cf025dc:36045 finished persisting bucket cache to file, renaming 2024-11-13T18:20:07,630 WARN [RS:2;fda96cf025dc:36045 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-13T18:20:07,630 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:20:07,630 INFO [RS:2;fda96cf025dc:36045 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:20:07,630 INFO [RS:2;fda96cf025dc:36045 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:20:07,630 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer(959): stopping server fda96cf025dc,36045,1731522006949 2024-11-13T18:20:07,630 INFO [RS:2;fda96cf025dc:36045 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:20:07,631 INFO [RS:2;fda96cf025dc:36045 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;fda96cf025dc:36045. 2024-11-13T18:20:07,631 DEBUG [RS:2;fda96cf025dc:36045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:20:07,631 DEBUG [RS:2;fda96cf025dc:36045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:07,631 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer(976): stopping server fda96cf025dc,36045,1731522006949; all regions closed. 2024-11-13T18:20:07,631 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:20:07,632 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T18:20:07,632 DEBUG [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 8d531c884d866a2490dfd2337a4db1a8=testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8.} 2024-11-13T18:20:07,633 DEBUG [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8d531c884d866a2490dfd2337a4db1a8 2024-11-13T18:20:07,633 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T18:20:07,633 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8d531c884d866a2490dfd2337a4db1a8, disabling compactions & flushes 2024-11-13T18:20:07,633 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:07,633 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:07,633 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. after waiting 0 ms 2024-11-13T18:20:07,633 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:07,634 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T18:20:07,634 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:20:07,634 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:20:07,634 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:20:07,634 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:20:07,635 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:20:07,635 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-13T18:20:07,637 DEBUG [StoreFileCloser-8d531c884d866a2490dfd2337a4db1a8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/family/9396570bcfce4c7881bcb3d9b78b7678 2024-11-13T18:20:07,642 DEBUG [RS:2;fda96cf025dc:36045 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs 2024-11-13T18:20:07,642 INFO [RS:2;fda96cf025dc:36045 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C36045%2C1731522006949:(num 1731522007197) 2024-11-13T18:20:07,642 DEBUG [RS:2;fda96cf025dc:36045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:07,642 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:20:07,643 INFO [RS:2;fda96cf025dc:36045 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:20:07,643 INFO [RS:2;fda96cf025dc:36045 {}] hbase.ChoreService(370): Chore service for: regionserver/fda96cf025dc:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T18:20:07,643 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:20:07,643 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:20:07,643 INFO [regionserver/fda96cf025dc:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:20:07,643 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:20:07,643 INFO [RS:2;fda96cf025dc:36045 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:20:07,643 INFO [RS:2;fda96cf025dc:36045 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36045 2024-11-13T18:20:07,645 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:20:07,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:20:07,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36045-0x1003a5f145f0003, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,36045,1731522006949 2024-11-13T18:20:07,650 INFO [RS:2;fda96cf025dc:36045 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:20:07,651 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007fb3004776e0@4555a36c rejected from java.util.concurrent.ThreadPoolExecutor@5735a2[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 4] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-13T18:20:07,652 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,36045,1731522006949] 2024-11-13T18:20:07,655 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,36045,1731522006949 already deleted, retry=false 2024-11-13T18:20:07,655 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fda96cf025dc,36045,1731522006949 expired; onlineServers=1 2024-11-13T18:20:07,669 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/default/testBlockEvictionOnGracefulStop1731522004738/8d531c884d866a2490dfd2337a4db1a8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-13T18:20:07,669 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:07,670 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8d531c884d866a2490dfd2337a4db1a8: Waiting for close lock at 1731522007632Running coprocessor pre-close hooks at 1731522007633 (+1 ms)Disabling compacts and flushes for region at 1731522007633Disabling writes for close at 1731522007633Writing region close event to WAL at 1731522007638 (+5 ms)Running coprocessor post-close hooks at 1731522007669 (+31 ms)Closed at 1731522007669 2024-11-13T18:20:07,670 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8. 2024-11-13T18:20:07,684 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 90c7a52715104692862edc0a50836562 2024-11-13T18:20:07,705 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/info/90c7a52715104692862edc0a50836562 is 209, key is testBlockEvictionOnGracefulStop1731522004738,,1731522004742.8d531c884d866a2490dfd2337a4db1a8./info:regioninfo/1731522007562/Put/seqid=0 2024-11-13T18:20:07,706 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/info/90c7a52715104692862edc0a50836562 2024-11-13T18:20:07,709 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/.tmp/info/90c7a52715104692862edc0a50836562 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/90c7a52715104692862edc0a50836562 2024-11-13T18:20:07,711 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/90c7a52715104692862edc0a50836562, delay=954 ms 2024-11-13T18:20:07,712 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/90c7a52715104692862edc0a50836562, entries=8, sequenceid=19, filesize=6.6 K 2024-11-13T18:20:07,713 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=19, compaction requested=false 2024-11-13T18:20:07,714 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/6f609e53064242bcb9e96b20f3a02bc0 2024-11-13T18:20:07,715 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/info/90c7a52715104692862edc0a50836562 2024-11-13T18:20:07,716 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/ns/5584919e553748a2a6a9ac6f9b05bc08 2024-11-13T18:20:07,717 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/table/df7640473dec46e3b9d667a9f8734234 2024-11-13T18:20:07,725 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-13T18:20:07,725 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:20:07,725 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:20:07,725 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731522007634Running coprocessor pre-close hooks at 1731522007634Disabling compacts and flushes for region at 1731522007634Disabling writes for close at 1731522007635 (+1 ms)Obtaining lock to block concurrent updates at 1731522007635Preparing flush snapshotting stores in 1588230740 at 1731522007635Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731522007635Flushing stores of hbase:meta,,1.1588230740 at 1731522007637 (+2 ms)Flushing 1588230740/info: creating writer at 1731522007637Flushing 1588230740/info: appending metadata at 1731522007704 (+67 ms)Flushing 1588230740/info: closing flushed file at 1731522007704Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@543b9316: reopening flushed file at 1731522007709 (+5 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=19, compaction requested=false at 1731522007714 (+5 ms)Writing region close event to WAL at 1731522007717 (+3 ms)Running coprocessor post-close hooks at 1731522007725 (+8 ms)Closed at 1731522007725 2024-11-13T18:20:07,725 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T18:20:07,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36045-0x1003a5f145f0003, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:07,754 INFO [RS:2;fda96cf025dc:36045 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:20:07,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36045-0x1003a5f145f0003, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:07,755 INFO [RS:2;fda96cf025dc:36045 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,36045,1731522006949; zookeeper connection closed. 2024-11-13T18:20:07,755 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3faf02bf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3faf02bf 2024-11-13T18:20:07,833 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(976): stopping server fda96cf025dc,40711,1731522003595; all regions closed. 2024-11-13T18:20:07,835 DEBUG [RS:1;fda96cf025dc:40711 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs 2024-11-13T18:20:07,836 INFO [RS:1;fda96cf025dc:40711 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C40711%2C1731522003595.meta:.meta(num 1731522007008) 2024-11-13T18:20:07,837 DEBUG [RS:1;fda96cf025dc:40711 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/oldWALs 2024-11-13T18:20:07,837 INFO [RS:1;fda96cf025dc:40711 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C40711%2C1731522003595:(num 1731522004150) 2024-11-13T18:20:07,837 DEBUG [RS:1;fda96cf025dc:40711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:20:07,838 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:20:07,838 INFO [RS:1;fda96cf025dc:40711 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:20:07,838 INFO [RS:1;fda96cf025dc:40711 {}] hbase.ChoreService(370): Chore service for: regionserver/fda96cf025dc:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T18:20:07,838 INFO [RS:1;fda96cf025dc:40711 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:20:07,838 INFO [regionserver/fda96cf025dc:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:20:07,838 INFO [RS:1;fda96cf025dc:40711 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40711 2024-11-13T18:20:07,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,40711,1731522003595 2024-11-13T18:20:07,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:20:07,841 INFO [RS:1;fda96cf025dc:40711 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:20:07,842 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,40711,1731522003595] 2024-11-13T18:20:07,844 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,40711,1731522003595 already deleted, retry=false 2024-11-13T18:20:07,844 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fda96cf025dc,40711,1731522003595 expired; onlineServers=0 2024-11-13T18:20:07,844 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fda96cf025dc,46067,1731522003442' ***** 2024-11-13T18:20:07,844 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T18:20:07,844 INFO [M:0;fda96cf025dc:46067 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:20:07,845 INFO [M:0;fda96cf025dc:46067 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:20:07,845 DEBUG [M:0;fda96cf025dc:46067 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T18:20:07,845 DEBUG [M:0;fda96cf025dc:46067 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T18:20:07,845 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T18:20:07,845 DEBUG [master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731522003900 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731522003900,5,FailOnTimeoutGroup] 2024-11-13T18:20:07,845 DEBUG [master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731522003899 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731522003899,5,FailOnTimeoutGroup] 2024-11-13T18:20:07,845 INFO [M:0;fda96cf025dc:46067 {}] hbase.ChoreService(370): Chore service for: master/fda96cf025dc:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T18:20:07,845 INFO [M:0;fda96cf025dc:46067 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:20:07,845 DEBUG [M:0;fda96cf025dc:46067 {}] master.HMaster(1795): Stopping service threads 2024-11-13T18:20:07,845 INFO [M:0;fda96cf025dc:46067 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T18:20:07,845 INFO [M:0;fda96cf025dc:46067 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:20:07,846 INFO [M:0;fda96cf025dc:46067 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:20:07,846 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T18:20:07,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T18:20:07,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:20:07,847 DEBUG [M:0;fda96cf025dc:46067 {}] zookeeper.ZKUtil(347): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T18:20:07,847 WARN [M:0;fda96cf025dc:46067 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T18:20:07,848 INFO [M:0;fda96cf025dc:46067 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/.lastflushedseqids 2024-11-13T18:20:07,854 INFO [M:0;fda96cf025dc:46067 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:20:07,855 INFO [M:0;fda96cf025dc:46067 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T18:20:07,855 DEBUG [M:0;fda96cf025dc:46067 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:20:07,855 INFO [M:0;fda96cf025dc:46067 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:07,855 DEBUG [M:0;fda96cf025dc:46067 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:07,855 DEBUG [M:0;fda96cf025dc:46067 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:20:07,855 DEBUG [M:0;fda96cf025dc:46067 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:07,855 INFO [M:0;fda96cf025dc:46067 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-13T18:20:07,866 DEBUG [M:0;fda96cf025dc:46067 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 420a9ba552714636967052dc82a961e5 2024-11-13T18:20:07,886 DEBUG [M:0;fda96cf025dc:46067 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/420a9ba552714636967052dc82a961e5 is 82, key is hbase:meta,,1/info:regioninfo/1731522007080/Put/seqid=0 2024-11-13T18:20:07,887 INFO [M:0;fda96cf025dc:46067 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/420a9ba552714636967052dc82a961e5 2024-11-13T18:20:07,908 DEBUG [M:0;fda96cf025dc:46067 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7f04f2f49ed346d4a3740138fbe52b39 2024-11-13T18:20:07,929 DEBUG [M:0;fda96cf025dc:46067 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7f04f2f49ed346d4a3740138fbe52b39 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731522005144/Put/seqid=0 2024-11-13T18:20:07,930 INFO [M:0;fda96cf025dc:46067 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7f04f2f49ed346d4a3740138fbe52b39 2024-11-13T18:20:07,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:07,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40711-0x1003a5f145f0002, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:07,944 INFO [RS:1;fda96cf025dc:40711 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:20:07,944 INFO [RS:1;fda96cf025dc:40711 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,40711,1731522003595; zookeeper connection closed. 2024-11-13T18:20:07,952 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2fb27d4c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2fb27d4c 2024-11-13T18:20:07,953 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T18:20:07,953 DEBUG [M:0;fda96cf025dc:46067 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0b3281078fa541f699b5889f20df40e7 2024-11-13T18:20:07,977 DEBUG [M:0;fda96cf025dc:46067 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0b3281078fa541f699b5889f20df40e7 is 69, key is fda96cf025dc,36045,1731522006949/rs:state/1731522007013/Put/seqid=0 2024-11-13T18:20:07,978 INFO [M:0;fda96cf025dc:46067 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0b3281078fa541f699b5889f20df40e7 2024-11-13T18:20:07,981 INFO [M:0;fda96cf025dc:46067 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0b3281078fa541f699b5889f20df40e7 2024-11-13T18:20:07,981 DEBUG [M:0;fda96cf025dc:46067 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/420a9ba552714636967052dc82a961e5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/420a9ba552714636967052dc82a961e5 2024-11-13T18:20:07,984 INFO [M:0;fda96cf025dc:46067 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/420a9ba552714636967052dc82a961e5, entries=14, sequenceid=102, filesize=5.9 K 2024-11-13T18:20:07,984 DEBUG [M:0;fda96cf025dc:46067 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7f04f2f49ed346d4a3740138fbe52b39 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7f04f2f49ed346d4a3740138fbe52b39 2024-11-13T18:20:07,987 INFO [M:0;fda96cf025dc:46067 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7f04f2f49ed346d4a3740138fbe52b39, entries=11, sequenceid=102, filesize=6.5 K 2024-11-13T18:20:07,987 DEBUG [M:0;fda96cf025dc:46067 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0b3281078fa541f699b5889f20df40e7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0b3281078fa541f699b5889f20df40e7 2024-11-13T18:20:07,989 INFO [M:0;fda96cf025dc:46067 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0b3281078fa541f699b5889f20df40e7 2024-11-13T18:20:07,990 INFO [M:0;fda96cf025dc:46067 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0dfbbc82-8062-ff28-8220-d8d0fbce3018/34e00daf-7933-41d8-6001-c77d383f7c15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0b3281078fa541f699b5889f20df40e7, entries=3, sequenceid=102, filesize=5.3 K 2024-11-13T18:20:07,991 INFO [M:0;fda96cf025dc:46067 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=102, compaction requested=false 2024-11-13T18:20:07,999 INFO [M:0;fda96cf025dc:46067 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:20:08,000 DEBUG [M:0;fda96cf025dc:46067 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731522007855Disabling compacts and flushes for region at 1731522007855Disabling writes for close at 1731522007855Obtaining lock to block concurrent updates at 1731522007855Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731522007855Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1731522007856 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731522007856Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731522007856Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731522007886 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731522007886Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731522007890 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731522007928 (+38 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731522007928Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731522007932 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731522007976 (+44 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731522007976Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@631542ed: reopening flushed file at 1731522007981 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b95aad8: reopening flushed file at 1731522007984 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24377361: reopening flushed file at 1731522007987 (+3 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=102, compaction requested=false at 1731522007991 (+4 ms)Writing region close event to WAL at 1731522007999 (+8 ms)Closed at 1731522007999 2024-11-13T18:20:08,001 INFO [M:0;fda96cf025dc:46067 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T18:20:08,001 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:20:08,001 INFO [M:0;fda96cf025dc:46067 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46067 2024-11-13T18:20:08,002 INFO [M:0;fda96cf025dc:46067 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:20:08,104 INFO [M:0;fda96cf025dc:46067 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:20:08,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:08,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46067-0x1003a5f145f0000, quorum=127.0.0.1:56021, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:20:08,108 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T18:20:08,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T18:20:08,109 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:20:08,118 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=616 (was 627), ProcessCount=11 (was 11), AvailableMemoryMB=3246 (was 3813)