2024-11-17 01:54:50,743 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-17 01:54:50,762 main DEBUG Took 0.016723 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 01:54:50,763 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 01:54:50,763 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 01:54:50,764 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 01:54:50,765 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,772 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 01:54:50,783 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,785 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,786 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,786 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,787 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,788 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,788 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,789 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,789 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,789 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,790 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,790 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,791 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,791 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,791 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,792 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,792 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,793 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,793 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,793 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,794 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:54:50,794 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,794 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 01:54:50,796 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:54:50,797 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 01:54:50,799 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 01:54:50,799 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 01:54:50,800 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 01:54:50,801 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 01:54:50,808 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 01:54:50,810 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 01:54:50,812 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 01:54:50,812 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 01:54:50,813 main DEBUG createAppenders(={Console}) 2024-11-17 01:54:50,813 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-17 01:54:50,814 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-17 01:54:50,814 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-17 01:54:50,814 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 01:54:50,815 main DEBUG OutputStream closed 2024-11-17 01:54:50,815 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 01:54:50,815 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 01:54:50,815 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-17 01:54:50,888 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 01:54:50,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 01:54:50,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 01:54:50,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 01:54:50,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 01:54:50,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 01:54:50,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 01:54:50,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 01:54:50,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 01:54:50,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 01:54:50,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 01:54:50,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 01:54:50,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 01:54:50,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 01:54:50,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 01:54:50,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 01:54:50,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 01:54:50,897 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 01:54:50,899 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 01:54:50,899 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-17 01:54:50,900 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 01:54:50,900 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-17T01:54:50,913 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-17 01:54:50,916 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 01:54:50,916 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T01:54:51,160 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87 2024-11-17T01:54:51,184 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=351, ProcessCount=11, AvailableMemoryMB=6472 2024-11-17T01:54:51,278 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T01:54:51,382 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:51,396 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/cluster_e51ec7f3-8ee9-b0f9-bc4f-6632634f605f, deleteOnExit=true 2024-11-17T01:54:51,534 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/cluster_e51ec7f3-8ee9-b0f9-bc4f-6632634f605f/zookeeper_0, clientPort=52935, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/cluster_e51ec7f3-8ee9-b0f9-bc4f-6632634f605f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/cluster_e51ec7f3-8ee9-b0f9-bc4f-6632634f605f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T01:54:51,545 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52935 2024-11-17T01:54:51,546 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:51,628 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8 with version=8 2024-11-17T01:54:51,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/hbase-staging 2024-11-17T01:54:51,720 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T01:54:51,943 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T01:54:51,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:51,954 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:51,965 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T01:54:51,965 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:51,966 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T01:54:52,109 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T01:54:52,184 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:54:52,217 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T01:54:52,226 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T01:54:52,230 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T01:54:52,266 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 94426 (auto-detected) 2024-11-17T01:54:52,267 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T01:54:52,289 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33301 2024-11-17T01:54:52,314 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33301 connecting to ZooKeeper ensemble=127.0.0.1:52935 2024-11-17T01:54:52,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:333010x0, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T01:54:52,404 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33301-0x10147ca0aba0000 connected 2024-11-17T01:54:52,487 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:52,489 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:52,499 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:54:52,503 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8, hbase.cluster.distributed=false 2024-11-17T01:54:52,527 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T01:54:52,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33301 2024-11-17T01:54:52,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33301 2024-11-17T01:54:52,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33301 2024-11-17T01:54:52,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33301 2024-11-17T01:54:52,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33301 2024-11-17T01:54:52,612 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T01:54:52,614 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:52,614 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:52,615 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T01:54:52,615 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:52,615 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T01:54:52,618 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T01:54:52,622 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T01:54:52,623 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33217 2024-11-17T01:54:52,625 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33217 connecting to ZooKeeper ensemble=127.0.0.1:52935 2024-11-17T01:54:52,626 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:52,627 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:52,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:332170x0, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T01:54:52,655 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:332170x0, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:54:52,655 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33217-0x10147ca0aba0001 connected 2024-11-17T01:54:52,665 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T01:54:52,680 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/bucket.cache 2024-11-17T01:54:52,689 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-17T01:54:52,691 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-17T01:54:52,693 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-17T01:54:52,696 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-17T01:54:52,696 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-17T01:54:52,706 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-17T01:54:52,708 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-17T01:54:52,708 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/bucket.persistence is complete. 2024-11-17T01:54:52,712 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T01:54:52,715 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T01:54:52,722 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T01:54:52,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33217 2024-11-17T01:54:52,726 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33217 2024-11-17T01:54:52,727 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33217 2024-11-17T01:54:52,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33217 2024-11-17T01:54:52,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33217 2024-11-17T01:54:52,759 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T01:54:52,760 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:52,760 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:52,761 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T01:54:52,761 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:52,761 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T01:54:52,762 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T01:54:52,762 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T01:54:52,763 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41717 2024-11-17T01:54:52,767 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41717 connecting to ZooKeeper ensemble=127.0.0.1:52935 2024-11-17T01:54:52,767 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:52,769 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:52,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417170x0, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T01:54:52,790 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:417170x0, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:54:52,790 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41717-0x10147ca0aba0002 connected 2024-11-17T01:54:52,791 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T01:54:52,793 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/bucket.cache 2024-11-17T01:54:52,793 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-17T01:54:52,793 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-17T01:54:52,794 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-17T01:54:52,794 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-17T01:54:52,795 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-17T01:54:52,796 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-17T01:54:52,796 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-17T01:54:52,796 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/bucket.persistence is complete. 2024-11-17T01:54:52,797 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T01:54:52,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T01:54:52,801 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T01:54:52,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41717 2024-11-17T01:54:52,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41717 2024-11-17T01:54:52,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41717 2024-11-17T01:54:52,808 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41717 2024-11-17T01:54:52,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41717 2024-11-17T01:54:52,825 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c6c0d7a0a7c0:33301 2024-11-17T01:54:52,826 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c6c0d7a0a7c0,33301,1731808491787 2024-11-17T01:54:52,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:52,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:52,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:52,856 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c6c0d7a0a7c0,33301,1731808491787 2024-11-17T01:54:52,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T01:54:52,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T01:54:52,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:52,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:52,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:52,888 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T01:54:52,889 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c6c0d7a0a7c0,33301,1731808491787 from backup master directory 2024-11-17T01:54:52,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:52,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:52,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c6c0d7a0a7c0,33301,1731808491787 2024-11-17T01:54:52,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:52,898 WARN [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T01:54:52,899 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c6c0d7a0a7c0,33301,1731808491787 2024-11-17T01:54:52,901 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T01:54:52,903 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T01:54:52,912 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/hbase.id] with ID: 3dca9ee4-df23-4b55-b79b-cdca0d6b97db 2024-11-17T01:54:52,912 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/.tmp/hbase.id 2024-11-17T01:54:52,917 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/hbase.id] 2024-11-17T01:54:52,951 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:52,958 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T01:54:52,977 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-17T01:54:53,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,035 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:54:53,039 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T01:54:53,080 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:53,085 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:54:53,146 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store 2024-11-17T01:54:53,165 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T01:54:53,168 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:53,169 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T01:54:53,169 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:53,169 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:53,171 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T01:54:53,171 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:53,171 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:53,172 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731808493169Disabling compacts and flushes for region at 1731808493169Disabling writes for close at 1731808493171 (+2 ms)Writing region close event to WAL at 1731808493171Closed at 1731808493171 2024-11-17T01:54:53,172 WARN [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/.initializing 2024-11-17T01:54:53,172 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/WALs/c6c0d7a0a7c0,33301,1731808491787 2024-11-17T01:54:53,184 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T01:54:53,196 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C33301%2C1731808491787, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/WALs/c6c0d7a0a7c0,33301,1731808491787, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/oldWALs, maxLogs=10 2024-11-17T01:54:53,229 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/WALs/c6c0d7a0a7c0,33301,1731808491787/c6c0d7a0a7c0%2C33301%2C1731808491787.1731808493200 2024-11-17T01:54:53,229 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T01:54:53,230 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:54:53,230 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:53,233 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,234 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,270 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,302 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T01:54:53,307 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:53,309 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:53,310 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,315 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T01:54:53,316 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:53,316 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:53,317 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,322 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T01:54:53,323 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:53,323 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:53,323 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,327 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T01:54:53,328 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:53,328 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:53,329 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,332 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,333 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,335 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,336 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,339 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T01:54:53,341 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:53,347 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:54:53,348 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69542242, jitterRate=0.03626015782356262}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T01:54:53,359 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731808493246Initializing all the Stores at 1731808493246Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808493247 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808493247Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808493248 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808493248Cleaning up temporary data from old regions at 1731808493336 (+88 ms)Region opened successfully at 1731808493358 (+22 ms) 2024-11-17T01:54:53,361 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T01:54:53,403 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1af41e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T01:54:53,449 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T01:54:53,467 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T01:54:53,467 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T01:54:53,471 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T01:54:53,472 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T01:54:53,480 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-17T01:54:53,481 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T01:54:53,507 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T01:54:53,515 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T01:54:53,547 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T01:54:53,550 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T01:54:53,552 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T01:54:53,562 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T01:54:53,564 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T01:54:53,568 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T01:54:53,579 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T01:54:53,580 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T01:54:53,589 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T01:54:53,607 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T01:54:53,614 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T01:54:53,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T01:54:53,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T01:54:53,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T01:54:53,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,634 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c6c0d7a0a7c0,33301,1731808491787, sessionid=0x10147ca0aba0000, setting cluster-up flag (Was=false) 2024-11-17T01:54:53,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,689 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T01:54:53,692 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6c0d7a0a7c0,33301,1731808491787 2024-11-17T01:54:53,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:53,739 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T01:54:53,741 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6c0d7a0a7c0,33301,1731808491787 2024-11-17T01:54:53,746 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T01:54:53,814 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(746): ClusterId : 3dca9ee4-df23-4b55-b79b-cdca0d6b97db 2024-11-17T01:54:53,814 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(746): ClusterId : 3dca9ee4-df23-4b55-b79b-cdca0d6b97db 2024-11-17T01:54:53,815 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T01:54:53,817 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T01:54:53,817 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T01:54:53,825 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T01:54:53,832 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T01:54:53,833 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T01:54:53,833 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T01:54:53,833 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T01:54:53,833 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T01:54:53,840 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T01:54:53,840 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T01:54:53,837 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c6c0d7a0a7c0,33301,1731808491787 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T01:54:53,841 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a369598, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T01:54:53,841 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7497a45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T01:54:53,845 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:54:53,846 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:54:53,846 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:54:53,846 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:54:53,846 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c6c0d7a0a7c0:0, corePoolSize=10, maxPoolSize=10 2024-11-17T01:54:53,846 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:53,846 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T01:54:53,847 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:53,850 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731808523850 2024-11-17T01:54:53,851 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:54:53,851 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T01:54:53,852 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T01:54:53,853 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T01:54:53,856 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c6c0d7a0a7c0:41717 2024-11-17T01:54:53,857 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T01:54:53,857 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T01:54:53,857 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:53,857 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T01:54:53,858 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T01:54:53,858 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c6c0d7a0a7c0:33217 2024-11-17T01:54:53,858 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T01:54:53,858 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:53,861 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T01:54:53,861 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T01:54:53,861 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T01:54:53,861 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T01:54:53,861 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T01:54:53,861 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T01:54:53,862 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T01:54:53,863 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T01:54:53,864 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,33301,1731808491787 with port=33217, startcode=1731808492579 2024-11-17T01:54:53,864 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T01:54:53,864 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,33301,1731808491787 with port=41717, startcode=1731808492759 2024-11-17T01:54:53,871 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T01:54:53,871 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T01:54:53,873 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808493872,5,FailOnTimeoutGroup] 2024-11-17T01:54:53,874 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808493873,5,FailOnTimeoutGroup] 2024-11-17T01:54:53,874 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:53,874 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T01:54:53,876 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:53,876 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T01:54:53,876 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T01:54:53,876 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:53,877 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T01:54:53,877 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8 2024-11-17T01:54:53,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:53,885 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-17T01:54:53,889 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T01:54:53,889 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:53,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:53,890 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-17T01:54:53,894 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T01:54:53,894 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:53,894 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:53,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-17T01:54:53,900 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T01:54:53,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:53,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:53,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-17T01:54:53,904 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T01:54:53,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:53,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:53,905 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T01:54:53,906 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740 2024-11-17T01:54:53,906 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740 2024-11-17T01:54:53,907 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T01:54:53,907 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T01:54:53,907 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T01:54:53,908 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T01:54:53,916 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:54:53,919 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75222924, jitterRate=0.12090891599655151}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T01:54:53,924 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731808493884Initializing all the Stores at 1731808493885 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808493885Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808493885Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808493885Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808493885Cleaning up temporary data from old regions at 1731808493907 (+22 ms)Region opened successfully at 1731808493924 (+17 ms) 2024-11-17T01:54:53,924 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T01:54:53,924 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T01:54:53,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T01:54:53,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T01:54:53,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T01:54:53,926 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T01:54:53,926 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731808493924Disabling compacts and flushes for region at 1731808493924Disabling writes for close at 1731808493925 (+1 ms)Writing region close event to WAL at 1731808493926 (+1 ms)Closed at 1731808493926 2024-11-17T01:54:53,928 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:54:53,929 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T01:54:53,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T01:54:53,944 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T01:54:53,945 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45449, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T01:54:53,945 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47721, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T01:54:53,947 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T01:54:53,952 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33301 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:53,953 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33301 {}] master.ServerManager(517): Registering regionserver=c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:53,961 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33301 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6c0d7a0a7c0,33217,1731808492579 2024-11-17T01:54:53,961 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33301 {}] master.ServerManager(517): Registering regionserver=c6c0d7a0a7c0,33217,1731808492579 2024-11-17T01:54:53,965 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8 2024-11-17T01:54:53,965 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8 2024-11-17T01:54:53,965 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-17T01:54:53,965 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-17T01:54:53,965 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T01:54:53,965 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T01:54:54,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T01:54:54,015 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] zookeeper.ZKUtil(111): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:54,015 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] zookeeper.ZKUtil(111): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6c0d7a0a7c0,33217,1731808492579 2024-11-17T01:54:54,015 WARN [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T01:54:54,015 WARN [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T01:54:54,016 INFO [RS:1;c6c0d7a0a7c0:41717 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:54:54,016 INFO [RS:0;c6c0d7a0a7c0:33217 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:54:54,016 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:54,016 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,33217,1731808492579 2024-11-17T01:54:54,019 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6c0d7a0a7c0,41717,1731808492759] 2024-11-17T01:54:54,020 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6c0d7a0a7c0,33217,1731808492579] 2024-11-17T01:54:54,042 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T01:54:54,042 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T01:54:54,061 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T01:54:54,061 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T01:54:54,066 INFO [RS:0;c6c0d7a0a7c0:33217 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T01:54:54,066 INFO [RS:1;c6c0d7a0a7c0:41717 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T01:54:54,066 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,066 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,067 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T01:54:54,068 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T01:54:54,072 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T01:54:54,072 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T01:54:54,073 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,073 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,074 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,074 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,074 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,074 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,074 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,074 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,074 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,074 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,074 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,074 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,074 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T01:54:54,074 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T01:54:54,074 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,075 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,075 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,075 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,075 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,075 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,075 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,075 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,075 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,075 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,075 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:54:54,075 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,075 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:54:54,076 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:54,076 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:54:54,076 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:54:54,082 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,082 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,082 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,082 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,082 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,083 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,083 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,083 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,083 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,083 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,083 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33217,1731808492579-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T01:54:54,083 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,41717,1731808492759-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T01:54:54,098 WARN [c6c0d7a0a7c0:33301 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:54:54,102 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T01:54:54,102 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T01:54:54,104 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33217,1731808492579-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,104 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,41717,1731808492759-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,104 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,104 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,104 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.Replication(171): c6c0d7a0a7c0,41717,1731808492759 started 2024-11-17T01:54:54,104 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.Replication(171): c6c0d7a0a7c0,33217,1731808492579 started 2024-11-17T01:54:54,124 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,124 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(1482): Serving as c6c0d7a0a7c0,33217,1731808492579, RpcServer on c6c0d7a0a7c0/172.17.0.2:33217, sessionid=0x10147ca0aba0001 2024-11-17T01:54:54,125 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T01:54:54,125 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6c0d7a0a7c0,33217,1731808492579 2024-11-17T01:54:54,125 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,33217,1731808492579' 2024-11-17T01:54:54,126 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,126 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T01:54:54,126 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(1482): Serving as c6c0d7a0a7c0,41717,1731808492759, RpcServer on c6c0d7a0a7c0/172.17.0.2:41717, sessionid=0x10147ca0aba0002 2024-11-17T01:54:54,126 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T01:54:54,126 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:54,126 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,41717,1731808492759' 2024-11-17T01:54:54,126 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T01:54:54,127 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T01:54:54,127 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T01:54:54,127 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T01:54:54,127 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T01:54:54,128 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6c0d7a0a7c0,33217,1731808492579 2024-11-17T01:54:54,128 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T01:54:54,128 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,33217,1731808492579' 2024-11-17T01:54:54,128 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T01:54:54,128 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:54,128 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T01:54:54,128 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,41717,1731808492759' 2024-11-17T01:54:54,128 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T01:54:54,128 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T01:54:54,128 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T01:54:54,129 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T01:54:54,129 INFO [RS:0;c6c0d7a0a7c0:33217 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T01:54:54,129 INFO [RS:0;c6c0d7a0a7c0:33217 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T01:54:54,129 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T01:54:54,129 INFO [RS:1;c6c0d7a0a7c0:41717 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T01:54:54,129 INFO [RS:1;c6c0d7a0a7c0:41717 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T01:54:54,238 INFO [RS:0;c6c0d7a0a7c0:33217 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T01:54:54,238 INFO [RS:1;c6c0d7a0a7c0:41717 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T01:54:54,239 INFO [RS:1;c6c0d7a0a7c0:41717 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c6c0d7a0a7c0%2C41717%2C1731808492759, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,41717,1731808492759, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs, maxLogs=55 2024-11-17T01:54:54,239 INFO [RS:0;c6c0d7a0a7c0:33217 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c6c0d7a0a7c0%2C33217%2C1731808492579, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,33217,1731808492579, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs, maxLogs=55 2024-11-17T01:54:54,258 INFO [RS:1;c6c0d7a0a7c0:41717 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,41717,1731808492759/c6c0d7a0a7c0%2C41717%2C1731808492759.1731808494242 2024-11-17T01:54:54,258 INFO [RS:0;c6c0d7a0a7c0:33217 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,33217,1731808492579/c6c0d7a0a7c0%2C33217%2C1731808492579.1731808494242 2024-11-17T01:54:54,259 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T01:54:54,259 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T01:54:54,354 DEBUG [c6c0d7a0a7c0:33301 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-17T01:54:54,366 DEBUG [c6c0d7a0a7c0:33301 {}] balancer.BalancerClusterState(204): Hosts are {c6c0d7a0a7c0=0} racks are {/default-rack=0} 2024-11-17T01:54:54,370 DEBUG [c6c0d7a0a7c0:33301 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T01:54:54,370 DEBUG [c6c0d7a0a7c0:33301 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T01:54:54,370 DEBUG [c6c0d7a0a7c0:33301 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T01:54:54,370 DEBUG [c6c0d7a0a7c0:33301 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T01:54:54,370 INFO [c6c0d7a0a7c0:33301 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T01:54:54,370 INFO [c6c0d7a0a7c0:33301 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T01:54:54,370 DEBUG [c6c0d7a0a7c0:33301 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T01:54:54,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:54,384 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,41717,1731808492759, state=OPENING 2024-11-17T01:54:54,422 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T01:54:54,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:54,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:54,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:54,432 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:54,432 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:54,432 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:54,433 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T01:54:54,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,41717,1731808492759}] 2024-11-17T01:54:54,607 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T01:54:54,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33215, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T01:54:54,619 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T01:54:54,619 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:54:54,620 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T01:54:54,621 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c6c0d7a0a7c0%2C41717%2C1731808492759.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,41717,1731808492759, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs, maxLogs=55 2024-11-17T01:54:54,641 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,41717,1731808492759/c6c0d7a0a7c0%2C41717%2C1731808492759.meta.1731808494623.meta 2024-11-17T01:54:54,641 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T01:54:54,642 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:54:54,644 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T01:54:54,646 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T01:54:54,651 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T01:54:54,655 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T01:54:54,656 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:54,656 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T01:54:54,656 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T01:54:54,658 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-17T01:54:54,659 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T01:54:54,659 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:54,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:54,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-17T01:54:54,660 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T01:54:54,660 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:54,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:54,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-17T01:54:54,661 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T01:54:54,661 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:54,661 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:54,661 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-17T01:54:54,662 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T01:54:54,662 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:54,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:54,663 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T01:54:54,664 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740 2024-11-17T01:54:54,664 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740 2024-11-17T01:54:54,665 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T01:54:54,665 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T01:54:54,665 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T01:54:54,666 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T01:54:54,667 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68937620, jitterRate=0.027250587940216064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T01:54:54,667 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T01:54:54,668 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731808494657Writing region info on filesystem at 1731808494657Initializing all the Stores at 1731808494657Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808494657Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808494658 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808494658Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808494658Cleaning up temporary data from old regions at 1731808494665 (+7 ms)Running coprocessor post-open hooks at 1731808494667 (+2 ms)Region opened successfully at 1731808494668 (+1 ms) 2024-11-17T01:54:54,673 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731808494599 2024-11-17T01:54:54,682 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T01:54:54,683 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T01:54:54,684 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:54,686 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,41717,1731808492759, state=OPEN 2024-11-17T01:54:54,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:54:54,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:54:54,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:54:54,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:54,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:54,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:54,713 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:54,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T01:54:54,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,41717,1731808492759 in 278 msec 2024-11-17T01:54:54,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T01:54:54,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 784 msec 2024-11-17T01:54:54,724 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:54:54,724 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T01:54:54,742 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:54,743 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41717,1731808492759, seqNum=-1] 2024-11-17T01:54:54,761 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:54,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45991, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:54,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0070 sec 2024-11-17T01:54:54,782 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731808494782, completionTime=-1 2024-11-17T01:54:54,785 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-17T01:54:54,785 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T01:54:54,810 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T01:54:54,810 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731808554810 2024-11-17T01:54:54,810 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731808614810 2024-11-17T01:54:54,811 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-17T01:54:54,813 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-17T01:54:54,820 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33301,1731808491787-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,820 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33301,1731808491787-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,820 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33301,1731808491787-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,821 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c6c0d7a0a7c0:33301, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,822 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,822 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,829 DEBUG [master/c6c0d7a0a7c0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T01:54:54,848 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.949sec 2024-11-17T01:54:54,850 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T01:54:54,851 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T01:54:54,852 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T01:54:54,853 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T01:54:54,853 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T01:54:54,854 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33301,1731808491787-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T01:54:54,854 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33301,1731808491787-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T01:54:54,855 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T01:54:54,856 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T01:54:54,856 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33301,1731808491787-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:54,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:54,932 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T01:54:54,932 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T01:54:54,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,33301,-1 for getting cluster id 2024-11-17T01:54:54,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:54,945 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dca9ee4-df23-4b55-b79b-cdca0d6b97db' 2024-11-17T01:54:54,947 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:54,947 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dca9ee4-df23-4b55-b79b-cdca0d6b97db" 2024-11-17T01:54:54,948 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3fa391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:54,948 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,33301,-1] 2024-11-17T01:54:54,950 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:54,951 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:54,953 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:54,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:54,956 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:54,962 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41717,1731808492759, seqNum=-1] 2024-11-17T01:54:54,963 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:54,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:54,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c6c0d7a0a7c0,33301,1731808491787 2024-11-17T01:54:54,999 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T01:54:55,003 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c6c0d7a0a7c0,33301,1731808491787 2024-11-17T01:54:55,006 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a760f1b 2024-11-17T01:54:55,007 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T01:54:55,012 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T01:54:55,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731808494994', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:54:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731808494994 2024-11-17T01:54:55,027 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731808494994 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:54:55,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731808494994" procId is: 4 2024-11-17T01:54:55,030 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:55,032 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731808494994 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:54:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:54:55,044 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731808494994-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8ac6885b7176c5183523d63e997761dd, NAME => 'testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731808494994', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8 2024-11-17T01:54:55,049 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731808494994-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:55,049 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731808494994-pool-0 {}] regionserver.HRegion(1722): Closing 8ac6885b7176c5183523d63e997761dd, disabling compactions & flushes 2024-11-17T01:54:55,049 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731808494994-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:55,049 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731808494994-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:55,049 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731808494994-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. after waiting 0 ms 2024-11-17T01:54:55,049 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731808494994-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:55,049 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731808494994-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:55,050 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731808494994-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8ac6885b7176c5183523d63e997761dd: Waiting for close lock at 1731808495049Disabling compacts and flushes for region at 1731808495049Disabling writes for close at 1731808495049Writing region close event to WAL at 1731808495049Closed at 1731808495049 2024-11-17T01:54:55,051 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731808494994 execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:54:55,056 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731808495051"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808495051"}]},"ts":"1731808495051"} 2024-11-17T01:54:55,060 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:54:55,061 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731808494994 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:54:55,063 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731808494994","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808495061"}]},"ts":"1731808495061"} 2024-11-17T01:54:55,067 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731808494994, state=ENABLING in hbase:meta 2024-11-17T01:54:55,068 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {c6c0d7a0a7c0=0} racks are {/default-rack=0} 2024-11-17T01:54:55,069 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T01:54:55,069 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T01:54:55,069 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T01:54:55,069 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T01:54:55,069 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T01:54:55,069 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T01:54:55,069 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T01:54:55,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731808494994, region=8ac6885b7176c5183523d63e997761dd, ASSIGN}] 2024-11-17T01:54:55,072 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731808494994, region=8ac6885b7176c5183523d63e997761dd, ASSIGN 2024-11-17T01:54:55,073 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731808494994, region=8ac6885b7176c5183523d63e997761dd, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,41717,1731808492759; forceNewPlan=false, retain=false 2024-11-17T01:54:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:54:55,226 INFO [c6c0d7a0a7c0:33301 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-17T01:54:55,226 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8ac6885b7176c5183523d63e997761dd, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:55,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731808494994, region=8ac6885b7176c5183523d63e997761dd, ASSIGN because future has completed 2024-11-17T01:54:55,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ac6885b7176c5183523d63e997761dd, server=c6c0d7a0a7c0,41717,1731808492759}] 2024-11-17T01:54:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:54:55,390 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:55,390 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ac6885b7176c5183523d63e997761dd, NAME => 'testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:54:55,391 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731808494994 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:55,392 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:55,392 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:55,392 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:55,394 INFO [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:55,400 INFO [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ac6885b7176c5183523d63e997761dd columnFamilyName family 2024-11-17T01:54:55,400 DEBUG [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:55,401 INFO [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] regionserver.HStore(327): Store=8ac6885b7176c5183523d63e997761dd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:55,401 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:55,402 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:55,402 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:55,402 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:55,402 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:55,403 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:55,408 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:54:55,408 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8ac6885b7176c5183523d63e997761dd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61994560, jitterRate=-0.07620906829833984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:54:55,408 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:55,409 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8ac6885b7176c5183523d63e997761dd: Running coprocessor pre-open hook at 1731808495393Writing region info on filesystem at 1731808495393Initializing all the Stores at 1731808495393Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808495393Cleaning up temporary data from old regions at 1731808495402 (+9 ms)Running coprocessor post-open hooks at 1731808495408 (+6 ms)Region opened successfully at 1731808495409 (+1 ms) 2024-11-17T01:54:55,410 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd., pid=6, masterSystemTime=1731808495385 2024-11-17T01:54:55,413 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:55,413 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:55,413 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8ac6885b7176c5183523d63e997761dd, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:55,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ac6885b7176c5183523d63e997761dd, server=c6c0d7a0a7c0,41717,1731808492759 because future has completed 2024-11-17T01:54:55,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T01:54:55,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ac6885b7176c5183523d63e997761dd, server=c6c0d7a0a7c0,41717,1731808492759 in 187 msec 2024-11-17T01:54:55,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T01:54:55,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731808494994, region=8ac6885b7176c5183523d63e997761dd, ASSIGN in 351 msec 2024-11-17T01:54:55,425 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731808494994 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:54:55,425 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731808494994","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808495425"}]},"ts":"1731808495425"} 2024-11-17T01:54:55,428 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731808494994, state=ENABLED in hbase:meta 2024-11-17T01:54:55,429 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731808494994 execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:54:55,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731808494994 in 407 msec 2024-11-17T01:54:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:54:55,662 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731808494994 get assigned. Timeout = 60000ms 2024-11-17T01:54:55,662 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731808494994 completed 2024-11-17T01:54:55,665 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:54:55,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731808494994 assigned to meta. Checking AM states. 2024-11-17T01:54:55,673 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:54:55,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731808494994 assigned. 2024-11-17T01:54:55,681 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731808494994', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd., hostname=c6c0d7a0a7c0,41717,1731808492759, seqNum=2] 2024-11-17T01:54:55,693 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8ac6885b7176c5183523d63e997761dd 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-17T01:54:55,716 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 194bc78a2d7441d9864002749f82eed0 2024-11-17T01:54:55,750 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/.tmp/family/194bc78a2d7441d9864002749f82eed0 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-17T01:54:55,755 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/.tmp/family/194bc78a2d7441d9864002749f82eed0 2024-11-17T01:54:55,784 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/.tmp/family/194bc78a2d7441d9864002749f82eed0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0 2024-11-17T01:54:55,785 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0, delay=938 ms 2024-11-17T01:54:55,787 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0, entries=2, sequenceid=6, filesize=4.9 K 2024-11-17T01:54:55,792 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 8ac6885b7176c5183523d63e997761dd in 100ms, sequenceid=6, compaction requested=false 2024-11-17T01:54:55,793 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731808494994' 2024-11-17T01:54:55,795 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8ac6885b7176c5183523d63e997761dd: 2024-11-17T01:54:56,726 DEBUG [hfile-prefetch-1731808495785 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 194bc78a2d7441d9864002749f82eed0_0. 2024-11-17T01:54:56,729 DEBUG [hfile-prefetch-1731808495785 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 194bc78a2d7441d9864002749f82eed0_117. 2024-11-17T01:54:56,730 DEBUG [hfile-prefetch-1731808495785 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0, with total blocks 2, and data blocks 1 2024-11-17T01:54:56,730 DEBUG [hfile-prefetch-1731808495785 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-17T01:54:56,731 DEBUG [hfile-prefetch-1731808495785 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-17T01:54:56,731 DEBUG [hfile-prefetch-1731808495785 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0. Will sleep 100ms and try the verification again. 2024-11-17T01:54:56,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8ac6885b7176c5183523d63e997761dd, source=c6c0d7a0a7c0,41717,1731808492759, destination=c6c0d7a0a7c0,33217,1731808492579, warming up region on c6c0d7a0a7c0,33217,1731808492579 2024-11-17T01:54:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T01:54:56,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8ac6885b7176c5183523d63e997761dd, source=c6c0d7a0a7c0,41717,1731808492759, destination=c6c0d7a0a7c0,33217,1731808492579, running balancer 2024-11-17T01:54:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731808494994, region=8ac6885b7176c5183523d63e997761dd, REOPEN/MOVE 2024-11-17T01:54:56,813 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731808494994, region=8ac6885b7176c5183523d63e997761dd, REOPEN/MOVE 2024-11-17T01:54:56,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54057, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T01:54:56,814 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8ac6885b7176c5183523d63e997761dd, regionState=CLOSING, regionLocation=c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:56,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33217 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33217 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8ac6885b7176c5183523d63e997761dd, NAME => 'testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:54:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33217 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:56,816 INFO [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:56,817 INFO [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ac6885b7176c5183523d63e997761dd columnFamilyName family 2024-11-17T01:54:56,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731808494994, region=8ac6885b7176c5183523d63e997761dd, REOPEN/MOVE because future has completed 2024-11-17T01:54:56,817 DEBUG [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:56,818 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-17T01:54:56,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ac6885b7176c5183523d63e997761dd, server=c6c0d7a0a7c0,41717,1731808492759}] 2024-11-17T01:54:56,823 DEBUG [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0 2024-11-17T01:54:56,823 INFO [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] regionserver.HStore(327): Store=8ac6885b7176c5183523d63e997761dd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33217 {}] regionserver.HRegion(1722): Closing 8ac6885b7176c5183523d63e997761dd, disabling compactions & flushes 2024-11-17T01:54:56,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33217 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33217 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33217 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. after waiting 0 ms 2024-11-17T01:54:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33217 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:56,825 DEBUG [StoreFileCloser-8ac6885b7176c5183523d63e997761dd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0 2024-11-17T01:54:56,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33217 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33217 {}] regionserver.HRegion(1676): Region close journal for 8ac6885b7176c5183523d63e997761dd: Waiting for close lock at 1731808496824Disabling compacts and flushes for region at 1731808496824Disabling writes for close at 1731808496824Writing region close event to WAL at 1731808496825 (+1 ms)Closed at 1731808496825 2024-11-17T01:54:56,827 WARN [hfile-prefetch-1731808495785 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-17T01:54:56,827 DEBUG [hfile-prefetch-1731808495785 {}] hfile.PrefetchExecutor(121): Prefetch completed for 194bc78a2d7441d9864002749f82eed0 2024-11-17T01:54:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33301 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-17T01:54:56,898 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-17T01:54:56,898 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-17T01:54:56,898 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-17T01:54:56,898 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-17T01:54:56,899 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T01:54:56,899 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T01:54:56,899 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T01:54:56,899 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T01:54:56,916 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T01:54:56,916 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731808496898 2024-11-17T01:54:56,916 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-17T01:54:56,919 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-17T01:54:56,919 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731808496898 2024-11-17T01:54:56,919 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-17T01:54:56,919 WARN [bucket-cache-persister {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-17T01:54:56,976 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:56,976 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-17T01:54:56,977 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 8ac6885b7176c5183523d63e997761dd, disabling compactions & flushes 2024-11-17T01:54:56,977 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:56,977 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:56,977 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. after waiting 0 ms 2024-11-17T01:54:56,977 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:56,978 DEBUG [StoreFileCloser-8ac6885b7176c5183523d63e997761dd-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 194bc78a2d7441d9864002749f82eed0, starting offset: 0, end offset: 9223372036854775807 2024-11-17T01:54:56,979 DEBUG [StoreFileCloser-8ac6885b7176c5183523d63e997761dd-family-1 {}] bucket.BucketCache(856): removed key 194bc78a2d7441d9864002749f82eed0_0 from back map with offset lock 2094080 in the evict process 2024-11-17T01:54:56,984 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-17T01:54:56,986 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:56,986 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 8ac6885b7176c5183523d63e997761dd: Waiting for close lock at 1731808496977Running coprocessor pre-close hooks at 1731808496977Disabling compacts and flushes for region at 1731808496977Disabling writes for close at 1731808496977Writing region close event to WAL at 1731808496980 (+3 ms)Running coprocessor post-close hooks at 1731808496985 (+5 ms)Closed at 1731808496986 (+1 ms) 2024-11-17T01:54:56,986 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 8ac6885b7176c5183523d63e997761dd move to c6c0d7a0a7c0,33217,1731808492579 record at close sequenceid=6 2024-11-17T01:54:56,990 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:56,991 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8ac6885b7176c5183523d63e997761dd, regionState=CLOSED 2024-11-17T01:54:56,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ac6885b7176c5183523d63e997761dd, server=c6c0d7a0a7c0,41717,1731808492759 because future has completed 2024-11-17T01:54:56,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T01:54:56,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 8ac6885b7176c5183523d63e997761dd, server=c6c0d7a0a7c0,41717,1731808492759 in 178 msec 2024-11-17T01:54:57,000 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731808494994, region=8ac6885b7176c5183523d63e997761dd, REOPEN/MOVE; state=CLOSED, location=c6c0d7a0a7c0,33217,1731808492579; forceNewPlan=false, retain=false 2024-11-17T01:54:57,016 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-17T01:54:57,016 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-17T01:54:57,017 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T01:54:57,017 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T01:54:57,019 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-17T01:54:57,019 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-17T01:54:57,019 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T01:54:57,019 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T01:54:57,019 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T01:54:57,019 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731808497016 2024-11-17T01:54:57,019 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-17T01:54:57,022 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T01:54:57,022 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731808497019 2024-11-17T01:54:57,022 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-17T01:54:57,150 INFO [c6c0d7a0a7c0:33301 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-17T01:54:57,151 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8ac6885b7176c5183523d63e997761dd, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33217,1731808492579 2024-11-17T01:54:57,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731808494994, region=8ac6885b7176c5183523d63e997761dd, REOPEN/MOVE because future has completed 2024-11-17T01:54:57,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ac6885b7176c5183523d63e997761dd, server=c6c0d7a0a7c0,33217,1731808492579}] 2024-11-17T01:54:57,314 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:57,314 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ac6885b7176c5183523d63e997761dd, NAME => 'testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:54:57,315 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731808494994 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,316 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:57,316 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,316 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,318 INFO [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,319 INFO [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ac6885b7176c5183523d63e997761dd columnFamilyName family 2024-11-17T01:54:57,319 DEBUG [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:57,321 DEBUG [StoreFileOpener-8ac6885b7176c5183523d63e997761dd-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0, delay=927 ms 2024-11-17T01:54:57,323 DEBUG [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0 2024-11-17T01:54:57,323 INFO [StoreOpener-8ac6885b7176c5183523d63e997761dd-1 {}] regionserver.HStore(327): Store=8ac6885b7176c5183523d63e997761dd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:57,324 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,325 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,325 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,325 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,325 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,330 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,331 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 8ac6885b7176c5183523d63e997761dd; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61112928, jitterRate=-0.08934640884399414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:54:57,331 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,332 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 8ac6885b7176c5183523d63e997761dd: Running coprocessor pre-open hook at 1731808497317Writing region info on filesystem at 1731808497317Initializing all the Stores at 1731808497317Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808497317Cleaning up temporary data from old regions at 1731808497326 (+9 ms)Running coprocessor post-open hooks at 1731808497331 (+5 ms)Region opened successfully at 1731808497332 (+1 ms) 2024-11-17T01:54:57,333 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd., pid=9, masterSystemTime=1731808497308 2024-11-17T01:54:57,336 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:57,336 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:57,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8ac6885b7176c5183523d63e997761dd, regionState=OPEN, openSeqNum=10, regionLocation=c6c0d7a0a7c0,33217,1731808492579 2024-11-17T01:54:57,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ac6885b7176c5183523d63e997761dd, server=c6c0d7a0a7c0,33217,1731808492579 because future has completed 2024-11-17T01:54:57,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-17T01:54:57,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ac6885b7176c5183523d63e997761dd, server=c6c0d7a0a7c0,33217,1731808492579 in 186 msec 2024-11-17T01:54:57,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731808494994, region=8ac6885b7176c5183523d63e997761dd, REOPEN/MOVE in 532 msec 2024-11-17T01:54:57,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T01:54:57,366 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:54:57,366 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:54:57,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:57,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:57,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:57,372 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T01:54:57,372 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-17T01:54:57,372 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c6c0d7a0a7c0,33301,1731808491787 2024-11-17T01:54:57,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T01:54:57,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T01:54:57,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T01:54:57,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:57,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:57,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:57,405 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T01:54:57,406 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:54:57,406 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:54:57,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:57,407 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:54:57,407 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6c0d7a0a7c0,33217,1731808492579' ***** 2024-11-17T01:54:57,407 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:54:57,408 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T01:54:57,408 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6c0d7a0a7c0,41717,1731808492759' ***** 2024-11-17T01:54:57,408 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T01:54:57,408 INFO [RS:0;c6c0d7a0a7c0:33217 {}] bucket.BucketCache(1738): Disabling cache 2024-11-17T01:54:57,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:54:57,409 INFO [RS:1;c6c0d7a0a7c0:41717 {}] bucket.BucketCache(1738): Disabling cache 2024-11-17T01:54:57,409 INFO [RS:0;c6c0d7a0a7c0:33217 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/bucket.persistence 2024-11-17T01:54:57,409 INFO [RS:1;c6c0d7a0a7c0:41717 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/bucket.persistence 2024-11-17T01:54:57,409 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-17T01:54:57,409 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-17T01:54:57,409 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-17T01:54:57,409 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-17T01:54:57,409 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-17T01:54:57,409 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-17T01:54:57,410 INFO [RS:0;c6c0d7a0a7c0:33217 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-17T01:54:57,410 INFO [RS:1;c6c0d7a0a7c0:41717 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-17T01:54:57,410 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-17T01:54:57,410 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-17T01:54:57,420 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] bucket.BucketCache(1487): Thread RS:0;c6c0d7a0a7c0:33217 started persisting bucket cache to file 2024-11-17T01:54:57,420 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] bucket.BucketCache(1487): Thread RS:1;c6c0d7a0a7c0:41717 started persisting bucket cache to file 2024-11-17T01:54:57,420 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T01:54:57,420 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T01:54:57,420 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T01:54:57,420 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T01:54:57,423 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T01:54:57,423 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T01:54:57,423 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731808497420 2024-11-17T01:54:57,423 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731808497420 2024-11-17T01:54:57,423 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] bucket.BucketCache(1509): Thread RS:1;c6c0d7a0a7c0:41717 finished persisting bucket cache to file, renaming 2024-11-17T01:54:57,423 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] bucket.BucketCache(1509): Thread RS:0;c6c0d7a0a7c0:33217 finished persisting bucket cache to file, renaming 2024-11-17T01:54:57,423 WARN [RS:0;c6c0d7a0a7c0:33217 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-17T01:54:57,423 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T01:54:57,423 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T01:54:57,423 INFO [RS:0;c6c0d7a0a7c0:33217 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T01:54:57,423 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T01:54:57,423 INFO [RS:1;c6c0d7a0a7c0:41717 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T01:54:57,423 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T01:54:57,423 INFO [RS:0;c6c0d7a0a7c0:33217 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T01:54:57,423 INFO [RS:1;c6c0d7a0a7c0:41717 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T01:54:57,423 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(959): stopping server c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:57,423 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(3091): Received CLOSE for 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,423 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T01:54:57,424 INFO [RS:1;c6c0d7a0a7c0:41717 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c6c0d7a0a7c0:41717. 2024-11-17T01:54:57,424 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:54:57,424 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:57,424 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(959): stopping server c6c0d7a0a7c0,33217,1731808492579 2024-11-17T01:54:57,424 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T01:54:57,424 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T01:54:57,424 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T01:54:57,424 INFO [RS:0;c6c0d7a0a7c0:33217 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c6c0d7a0a7c0:33217. 2024-11-17T01:54:57,424 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T01:54:57,424 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:54:57,424 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T01:54:57,424 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:57,424 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T01:54:57,424 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-17T01:54:57,424 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(1325): Online Regions={8ac6885b7176c5183523d63e997761dd=testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd.} 2024-11-17T01:54:57,425 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T01:54:57,425 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T01:54:57,425 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-17T01:54:57,425 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8ac6885b7176c5183523d63e997761dd, disabling compactions & flushes 2024-11-17T01:54:57,425 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T01:54:57,425 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:57,425 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T01:54:57,425 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(1351): Waiting on 8ac6885b7176c5183523d63e997761dd 2024-11-17T01:54:57,425 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T01:54:57,425 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:57,425 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T01:54:57,425 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. after waiting 0 ms 2024-11-17T01:54:57,425 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T01:54:57,425 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:57,425 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T01:54:57,425 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-17T01:54:57,426 DEBUG [StoreFileCloser-8ac6885b7176c5183523d63e997761dd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/family/194bc78a2d7441d9864002749f82eed0 2024-11-17T01:54:57,430 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnRegionMove1731808494994/8ac6885b7176c5183523d63e997761dd/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-17T01:54:57,430 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:57,431 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8ac6885b7176c5183523d63e997761dd: Waiting for close lock at 1731808497424Running coprocessor pre-close hooks at 1731808497425 (+1 ms)Disabling compacts and flushes for region at 1731808497425Disabling writes for close at 1731808497425Writing region close event to WAL at 1731808497426 (+1 ms)Running coprocessor post-close hooks at 1731808497430 (+4 ms)Closed at 1731808497430 2024-11-17T01:54:57,431 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd. 2024-11-17T01:54:57,435 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a2608e97e3384389a2dc782921d70662 2024-11-17T01:54:57,455 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/info/a2608e97e3384389a2dc782921d70662 is 205, key is testBlockEvictionOnRegionMove1731808494994,,1731808495014.8ac6885b7176c5183523d63e997761dd./info:regioninfo/1731808497336/Put/seqid=0 2024-11-17T01:54:57,456 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/info/a2608e97e3384389a2dc782921d70662 2024-11-17T01:54:57,464 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cada93ce6ab34469bbfa45722d4d642e 2024-11-17T01:54:57,476 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/ns/cada93ce6ab34469bbfa45722d4d642e is 43, key is default/ns:d/1731808494767/Put/seqid=0 2024-11-17T01:54:57,477 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/ns/cada93ce6ab34469bbfa45722d4d642e 2024-11-17T01:54:57,485 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5aa25cbee32943fc9ee2f2d3cdf94f95 2024-11-17T01:54:57,495 INFO [regionserver/c6c0d7a0a7c0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:54:57,495 INFO [regionserver/c6c0d7a0a7c0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:54:57,498 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/table/5aa25cbee32943fc9ee2f2d3cdf94f95 is 78, key is testBlockEvictionOnRegionMove1731808494994/table:state/1731808495425/Put/seqid=0 2024-11-17T01:54:57,498 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/table/5aa25cbee32943fc9ee2f2d3cdf94f95 2024-11-17T01:54:57,501 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/info/a2608e97e3384389a2dc782921d70662 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/a2608e97e3384389a2dc782921d70662 2024-11-17T01:54:57,502 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/a2608e97e3384389a2dc782921d70662, delay=939 ms 2024-11-17T01:54:57,503 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/a2608e97e3384389a2dc782921d70662, entries=15, sequenceid=15, filesize=7.6 K 2024-11-17T01:54:57,504 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/ns/cada93ce6ab34469bbfa45722d4d642e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/ns/cada93ce6ab34469bbfa45722d4d642e 2024-11-17T01:54:57,505 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/ns/cada93ce6ab34469bbfa45722d4d642e, delay=929 ms 2024-11-17T01:54:57,506 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/ns/cada93ce6ab34469bbfa45722d4d642e, entries=2, sequenceid=15, filesize=5.0 K 2024-11-17T01:54:57,507 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/table/5aa25cbee32943fc9ee2f2d3cdf94f95 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/table/5aa25cbee32943fc9ee2f2d3cdf94f95 2024-11-17T01:54:57,508 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/table/5aa25cbee32943fc9ee2f2d3cdf94f95, delay=950 ms 2024-11-17T01:54:57,509 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/table/5aa25cbee32943fc9ee2f2d3cdf94f95, entries=2, sequenceid=15, filesize=5.3 K 2024-11-17T01:54:57,510 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=15, compaction requested=false 2024-11-17T01:54:57,510 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T01:54:57,511 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/a2608e97e3384389a2dc782921d70662 2024-11-17T01:54:57,512 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/ns/cada93ce6ab34469bbfa45722d4d642e 2024-11-17T01:54:57,512 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/table/5aa25cbee32943fc9ee2f2d3cdf94f95 2024-11-17T01:54:57,518 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-17T01:54:57,519 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T01:54:57,519 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T01:54:57,520 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731808497425Running coprocessor pre-close hooks at 1731808497425Disabling compacts and flushes for region at 1731808497425Disabling writes for close at 1731808497425Obtaining lock to block concurrent updates at 1731808497425Preparing flush snapshotting stores in 1588230740 at 1731808497425Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731808497426 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731808497426Flushing 1588230740/info: creating writer at 1731808497427 (+1 ms)Flushing 1588230740/info: appending metadata at 1731808497454 (+27 ms)Flushing 1588230740/info: closing flushed file at 1731808497454Flushing 1588230740/ns: creating writer at 1731808497458 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731808497476 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731808497476Flushing 1588230740/table: creating writer at 1731808497480 (+4 ms)Flushing 1588230740/table: appending metadata at 1731808497497 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731808497497Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69fad436: reopening flushed file at 1731808497501 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a4531a4: reopening flushed file at 1731808497504 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59781cac: reopening flushed file at 1731808497506 (+2 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=15, compaction requested=false at 1731808497510 (+4 ms)Writing region close event to WAL at 1731808497513 (+3 ms)Running coprocessor post-close hooks at 1731808497519 (+6 ms)Closed at 1731808497519 2024-11-17T01:54:57,520 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T01:54:57,625 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(976): stopping server c6c0d7a0a7c0,33217,1731808492579; all regions closed. 2024-11-17T01:54:57,625 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(976): stopping server c6c0d7a0a7c0,41717,1731808492759; all regions closed. 2024-11-17T01:54:57,632 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs 2024-11-17T01:54:57,632 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs 2024-11-17T01:54:57,632 INFO [RS:0;c6c0d7a0a7c0:33217 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C33217%2C1731808492579:(num 1731808494242) 2024-11-17T01:54:57,632 INFO [RS:1;c6c0d7a0a7c0:41717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C41717%2C1731808492759.meta:.meta(num 1731808494623) 2024-11-17T01:54:57,632 DEBUG [RS:0;c6c0d7a0a7c0:33217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:57,632 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:54:57,632 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T01:54:57,633 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.ChoreService(370): Chore service for: regionserver/c6c0d7a0a7c0:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T01:54:57,633 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T01:54:57,633 INFO [regionserver/c6c0d7a0a7c0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T01:54:57,633 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T01:54:57,634 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T01:54:57,634 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T01:54:57,634 INFO [RS:0;c6c0d7a0a7c0:33217 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33217 2024-11-17T01:54:57,635 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs 2024-11-17T01:54:57,635 INFO [RS:1;c6c0d7a0a7c0:41717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C41717%2C1731808492759:(num 1731808494242) 2024-11-17T01:54:57,635 DEBUG [RS:1;c6c0d7a0a7c0:41717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:57,635 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:54:57,635 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T01:54:57,635 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.ChoreService(370): Chore service for: regionserver/c6c0d7a0a7c0:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T01:54:57,636 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T01:54:57,636 INFO [regionserver/c6c0d7a0a7c0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T01:54:57,636 INFO [RS:1;c6c0d7a0a7c0:41717 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41717 2024-11-17T01:54:57,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T01:54:57,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6c0d7a0a7c0,33217,1731808492579 2024-11-17T01:54:57,661 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T01:54:57,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6c0d7a0a7c0,41717,1731808492759 2024-11-17T01:54:57,661 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T01:54:57,662 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6c0d7a0a7c0,41717,1731808492759] 2024-11-17T01:54:57,679 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6c0d7a0a7c0,41717,1731808492759 already deleted, retry=false 2024-11-17T01:54:57,680 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c6c0d7a0a7c0,41717,1731808492759 expired; onlineServers=1 2024-11-17T01:54:57,680 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6c0d7a0a7c0,33217,1731808492579] 2024-11-17T01:54:57,688 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6c0d7a0a7c0,33217,1731808492579 already deleted, retry=false 2024-11-17T01:54:57,688 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c6c0d7a0a7c0,33217,1731808492579 expired; onlineServers=0 2024-11-17T01:54:57,688 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c6c0d7a0a7c0,33301,1731808491787' ***** 2024-11-17T01:54:57,688 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T01:54:57,688 INFO [M:0;c6c0d7a0a7c0:33301 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T01:54:57,688 INFO [M:0;c6c0d7a0a7c0:33301 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T01:54:57,688 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T01:54:57,689 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T01:54:57,689 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T01:54:57,689 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808493872 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808493872,5,FailOnTimeoutGroup] 2024-11-17T01:54:57,689 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808493873 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808493873,5,FailOnTimeoutGroup] 2024-11-17T01:54:57,689 INFO [M:0;c6c0d7a0a7c0:33301 {}] hbase.ChoreService(370): Chore service for: master/c6c0d7a0a7c0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T01:54:57,689 INFO [M:0;c6c0d7a0a7c0:33301 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T01:54:57,689 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] master.HMaster(1795): Stopping service threads 2024-11-17T01:54:57,689 INFO [M:0;c6c0d7a0a7c0:33301 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T01:54:57,689 INFO [M:0;c6c0d7a0a7c0:33301 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T01:54:57,690 INFO [M:0;c6c0d7a0a7c0:33301 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T01:54:57,690 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T01:54:57,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T01:54:57,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:57,696 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] zookeeper.ZKUtil(347): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T01:54:57,696 WARN [M:0;c6c0d7a0a7c0:33301 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T01:54:57,697 INFO [M:0;c6c0d7a0a7c0:33301 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/.lastflushedseqids 2024-11-17T01:54:57,704 INFO [M:0;c6c0d7a0a7c0:33301 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T01:54:57,705 INFO [M:0;c6c0d7a0a7c0:33301 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T01:54:57,705 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T01:54:57,705 INFO [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:57,705 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:57,705 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T01:54:57,705 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:57,705 INFO [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-17T01:54:57,713 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 08fe114dc875463baf74f877a5a6e0bf 2024-11-17T01:54:57,726 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/08fe114dc875463baf74f877a5a6e0bf is 82, key is hbase:meta,,1/info:regioninfo/1731808494683/Put/seqid=0 2024-11-17T01:54:57,726 INFO [M:0;c6c0d7a0a7c0:33301 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/08fe114dc875463baf74f877a5a6e0bf 2024-11-17T01:54:57,734 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aa31d22e1aba44fca02f50b025fd755f 2024-11-17T01:54:57,747 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa31d22e1aba44fca02f50b025fd755f is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731808495431/Put/seqid=0 2024-11-17T01:54:57,748 INFO [M:0;c6c0d7a0a7c0:33301 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa31d22e1aba44fca02f50b025fd755f 2024-11-17T01:54:57,755 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on eb81e6b8160b4e99bee95a7d7a997676 2024-11-17T01:54:57,767 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eb81e6b8160b4e99bee95a7d7a997676 is 69, key is c6c0d7a0a7c0,33217,1731808492579/rs:state/1731808493962/Put/seqid=0 2024-11-17T01:54:57,767 INFO [M:0;c6c0d7a0a7c0:33301 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eb81e6b8160b4e99bee95a7d7a997676 2024-11-17T01:54:57,769 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/08fe114dc875463baf74f877a5a6e0bf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/08fe114dc875463baf74f877a5a6e0bf 2024-11-17T01:54:57,771 INFO [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/08fe114dc875463baf74f877a5a6e0bf, entries=8, sequenceid=84, filesize=5.5 K 2024-11-17T01:54:57,771 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa31d22e1aba44fca02f50b025fd755f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aa31d22e1aba44fca02f50b025fd755f 2024-11-17T01:54:57,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:54:57,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:54:57,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41717-0x10147ca0aba0002, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:54:57,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33217-0x10147ca0aba0001, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:54:57,772 INFO [RS:1;c6c0d7a0a7c0:41717 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T01:54:57,772 INFO [RS:0;c6c0d7a0a7c0:33217 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T01:54:57,772 INFO [RS:0;c6c0d7a0a7c0:33217 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6c0d7a0a7c0,33217,1731808492579; zookeeper connection closed. 2024-11-17T01:54:57,772 INFO [RS:1;c6c0d7a0a7c0:41717 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6c0d7a0a7c0,41717,1731808492759; zookeeper connection closed. 2024-11-17T01:54:57,773 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c68e63 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c68e63 2024-11-17T01:54:57,773 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c7fc797 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c7fc797 2024-11-17T01:54:57,773 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-17T01:54:57,773 INFO [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aa31d22e1aba44fca02f50b025fd755f, entries=9, sequenceid=84, filesize=6.5 K 2024-11-17T01:54:57,774 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eb81e6b8160b4e99bee95a7d7a997676 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eb81e6b8160b4e99bee95a7d7a997676 2024-11-17T01:54:57,775 INFO [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eb81e6b8160b4e99bee95a7d7a997676, entries=2, sequenceid=84, filesize=5.1 K 2024-11-17T01:54:57,776 INFO [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=84, compaction requested=false 2024-11-17T01:54:57,778 INFO [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:57,778 DEBUG [M:0;c6c0d7a0a7c0:33301 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731808497705Disabling compacts and flushes for region at 1731808497705Disabling writes for close at 1731808497705Obtaining lock to block concurrent updates at 1731808497705Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731808497705Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1731808497706 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731808497706Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731808497706Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731808497725 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731808497725Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731808497728 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731808497746 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731808497746Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731808497749 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731808497766 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731808497766Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2ee87e: reopening flushed file at 1731808497769 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a80c17b: reopening flushed file at 1731808497771 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c24cb7d: reopening flushed file at 1731808497773 (+2 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=84, compaction requested=false at 1731808497776 (+3 ms)Writing region close event to WAL at 1731808497778 (+2 ms)Closed at 1731808497778 2024-11-17T01:54:57,780 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T01:54:57,780 INFO [M:0;c6c0d7a0a7c0:33301 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T01:54:57,780 INFO [M:0;c6c0d7a0a7c0:33301 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33301 2024-11-17T01:54:57,780 INFO [M:0;c6c0d7a0a7c0:33301 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T01:54:57,905 INFO [M:0;c6c0d7a0a7c0:33301 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T01:54:57,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:54:57,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33301-0x10147ca0aba0000, quorum=127.0.0.1:52935, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:54:57,913 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T01:54:57,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-17T01:54:57,933 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:57,940 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: master/c6c0d7a0a7c0:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/c6c0d7a0a7c0:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731808497502 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/c6c0d7a0a7c0:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/c6c0d7a0a7c0:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731808497505 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731808497322 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731808495785 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=311 (was 351), ProcessCount=11 (was 11), AvailableMemoryMB=6126 (was 6472) 2024-11-17T01:54:57,945 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=311, ProcessCount=11, AvailableMemoryMB=6126 2024-11-17T01:54:57,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/hadoop-log-dir so I do NOT create it in target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9 2024-11-17T01:54:57,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/hadoop-tmp-dir so I do NOT create it in target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9 2024-11-17T01:54:57,946 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:57,949 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/cluster_bfb5e66d-b827-6479-6aac-d7b13af32a15, deleteOnExit=true 2024-11-17T01:54:57,977 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/cluster_bfb5e66d-b827-6479-6aac-d7b13af32a15/zookeeper_0, clientPort=49311, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/cluster_bfb5e66d-b827-6479-6aac-d7b13af32a15/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/cluster_bfb5e66d-b827-6479-6aac-d7b13af32a15/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T01:54:57,978 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49311 2024-11-17T01:54:57,991 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8 with version=8 2024-11-17T01:54:57,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/hbase-staging 2024-11-17T01:54:57,993 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T01:54:57,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:57,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:57,994 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T01:54:57,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:57,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T01:54:57,994 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T01:54:57,994 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T01:54:57,994 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38497 2024-11-17T01:54:57,996 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38497 connecting to ZooKeeper ensemble=127.0.0.1:49311 2024-11-17T01:54:58,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:384970x0, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T01:54:58,037 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38497-0x10147ca23fd0000 connected 2024-11-17T01:54:58,102 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:58,103 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:58,105 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:54:58,106 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8, hbase.cluster.distributed=false 2024-11-17T01:54:58,109 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T01:54:58,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38497 2024-11-17T01:54:58,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38497 2024-11-17T01:54:58,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38497 2024-11-17T01:54:58,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38497 2024-11-17T01:54:58,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38497 2024-11-17T01:54:58,126 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T01:54:58,126 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:58,126 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:58,126 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T01:54:58,126 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:58,126 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T01:54:58,126 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T01:54:58,127 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T01:54:58,127 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39257 2024-11-17T01:54:58,129 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39257 connecting to ZooKeeper ensemble=127.0.0.1:49311 2024-11-17T01:54:58,129 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:58,130 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:58,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:392570x0, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T01:54:58,135 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39257-0x10147ca23fd0001 connected 2024-11-17T01:54:58,135 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:54:58,136 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T01:54:58,136 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.cache 2024-11-17T01:54:58,136 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-17T01:54:58,136 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-17T01:54:58,136 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-17T01:54:58,137 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-17T01:54:58,137 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-17T01:54:58,137 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-17T01:54:58,137 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-17T01:54:58,137 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.persistence is complete. 2024-11-17T01:54:58,138 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T01:54:58,139 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T01:54:58,141 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T01:54:58,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39257 2024-11-17T01:54:58,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39257 2024-11-17T01:54:58,142 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39257 2024-11-17T01:54:58,142 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39257 2024-11-17T01:54:58,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39257 2024-11-17T01:54:58,161 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T01:54:58,161 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:58,161 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:58,161 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T01:54:58,161 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:54:58,161 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T01:54:58,161 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T01:54:58,161 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T01:54:58,162 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34897 2024-11-17T01:54:58,163 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34897 connecting to ZooKeeper ensemble=127.0.0.1:49311 2024-11-17T01:54:58,164 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:58,164 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:58,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348970x0, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T01:54:58,177 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:348970x0, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:54:58,177 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34897-0x10147ca23fd0002 connected 2024-11-17T01:54:58,177 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T01:54:58,178 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.cache 2024-11-17T01:54:58,178 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-17T01:54:58,179 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-17T01:54:58,179 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-17T01:54:58,179 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-17T01:54:58,179 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-17T01:54:58,180 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-17T01:54:58,180 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.persistence is complete. 2024-11-17T01:54:58,180 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-17T01:54:58,180 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T01:54:58,181 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T01:54:58,183 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T01:54:58,186 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34897 2024-11-17T01:54:58,186 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34897 2024-11-17T01:54:58,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34897 2024-11-17T01:54:58,191 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34897 2024-11-17T01:54:58,191 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34897 2024-11-17T01:54:58,202 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c6c0d7a0a7c0:38497 2024-11-17T01:54:58,203 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c6c0d7a0a7c0,38497,1731808497993 2024-11-17T01:54:58,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:58,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:58,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:58,210 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c6c0d7a0a7c0,38497,1731808497993 2024-11-17T01:54:58,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T01:54:58,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T01:54:58,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,219 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T01:54:58,220 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c6c0d7a0a7c0,38497,1731808497993 from backup master directory 2024-11-17T01:54:58,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:58,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:58,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c6c0d7a0a7c0,38497,1731808497993 2024-11-17T01:54:58,226 WARN [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T01:54:58,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:54:58,226 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c6c0d7a0a7c0,38497,1731808497993 2024-11-17T01:54:58,228 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/hbase.id] with ID: 1f97daa1-46f2-4024-a781-9f7075a910d8 2024-11-17T01:54:58,228 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/.tmp/hbase.id 2024-11-17T01:54:58,231 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/hbase.id] 2024-11-17T01:54:58,260 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:54:58,260 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T01:54:58,263 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-17T01:54:58,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,273 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:54:58,274 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T01:54:58,274 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:54:58,294 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store 2024-11-17T01:54:58,302 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:58,302 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T01:54:58,302 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:58,302 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:58,302 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T01:54:58,303 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:58,303 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:58,303 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731808498302Disabling compacts and flushes for region at 1731808498302Disabling writes for close at 1731808498303 (+1 ms)Writing region close event to WAL at 1731808498303Closed at 1731808498303 2024-11-17T01:54:58,303 WARN [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/.initializing 2024-11-17T01:54:58,303 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/WALs/c6c0d7a0a7c0,38497,1731808497993 2024-11-17T01:54:58,309 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T01:54:58,313 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C38497%2C1731808497993, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/WALs/c6c0d7a0a7c0,38497,1731808497993, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/oldWALs, maxLogs=10 2024-11-17T01:54:58,329 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/WALs/c6c0d7a0a7c0,38497,1731808497993/c6c0d7a0a7c0%2C38497%2C1731808497993.1731808498313 2024-11-17T01:54:58,329 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T01:54:58,330 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:54:58,330 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:58,330 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,330 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,331 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,334 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T01:54:58,335 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:58,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:58,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T01:54:58,338 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:58,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:58,339 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T01:54:58,342 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:58,342 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:58,342 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T01:54:58,345 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:58,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:58,345 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,346 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,346 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,346 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,346 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,347 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T01:54:58,347 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:54:58,352 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:54:58,352 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74487077, jitterRate=0.1099439412355423}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T01:54:58,353 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731808498330Initializing all the Stores at 1731808498330Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808498330Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808498331 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808498331Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808498331Cleaning up temporary data from old regions at 1731808498346 (+15 ms)Region opened successfully at 1731808498353 (+7 ms) 2024-11-17T01:54:58,354 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T01:54:58,358 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25db26cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T01:54:58,359 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T01:54:58,359 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T01:54:58,359 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T01:54:58,359 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T01:54:58,360 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T01:54:58,360 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-17T01:54:58,360 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T01:54:58,362 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T01:54:58,363 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T01:54:58,376 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T01:54:58,377 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T01:54:58,378 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T01:54:58,385 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T01:54:58,385 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T01:54:58,387 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T01:54:58,393 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T01:54:58,394 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T01:54:58,401 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T01:54:58,405 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T01:54:58,409 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T01:54:58,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T01:54:58,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T01:54:58,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T01:54:58,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,419 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c6c0d7a0a7c0,38497,1731808497993, sessionid=0x10147ca23fd0000, setting cluster-up flag (Was=false) 2024-11-17T01:54:58,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,460 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T01:54:58,461 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6c0d7a0a7c0,38497,1731808497993 2024-11-17T01:54:58,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:58,501 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T01:54:58,505 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6c0d7a0a7c0,38497,1731808497993 2024-11-17T01:54:58,507 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T01:54:58,508 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T01:54:58,509 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T01:54:58,509 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T01:54:58,509 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c6c0d7a0a7c0,38497,1731808497993 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T01:54:58,512 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:54:58,512 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:54:58,512 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:54:58,512 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:54:58,512 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c6c0d7a0a7c0:0, corePoolSize=10, maxPoolSize=10 2024-11-17T01:54:58,512 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,512 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T01:54:58,513 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,516 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:54:58,516 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T01:54:58,516 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:58,517 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T01:54:58,520 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731808528520 2024-11-17T01:54:58,520 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T01:54:58,521 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T01:54:58,521 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T01:54:58,521 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T01:54:58,521 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T01:54:58,521 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T01:54:58,521 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,523 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T01:54:58,523 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T01:54:58,523 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T01:54:58,526 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T01:54:58,527 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T01:54:58,528 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808498527,5,FailOnTimeoutGroup] 2024-11-17T01:54:58,530 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808498529,5,FailOnTimeoutGroup] 2024-11-17T01:54:58,530 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,531 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T01:54:58,531 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,531 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,537 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T01:54:58,537 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8 2024-11-17T01:54:58,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:58,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-17T01:54:58,550 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T01:54:58,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:58,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:58,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-17T01:54:58,553 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T01:54:58,553 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:58,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:58,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-17T01:54:58,557 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T01:54:58,557 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:58,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:58,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-17T01:54:58,560 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T01:54:58,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:58,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:58,561 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T01:54:58,561 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740 2024-11-17T01:54:58,561 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740 2024-11-17T01:54:58,562 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T01:54:58,562 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T01:54:58,562 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T01:54:58,562 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T01:54:58,568 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:54:58,568 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75468347, jitterRate=0.12456600368022919}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T01:54:58,569 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731808498547Initializing all the Stores at 1731808498547Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808498547Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808498547Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808498547Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808498547Cleaning up temporary data from old regions at 1731808498562 (+15 ms)Region opened successfully at 1731808498569 (+7 ms) 2024-11-17T01:54:58,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T01:54:58,570 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T01:54:58,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T01:54:58,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T01:54:58,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T01:54:58,570 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T01:54:58,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731808498569Disabling compacts and flushes for region at 1731808498569Disabling writes for close at 1731808498570 (+1 ms)Writing region close event to WAL at 1731808498570Closed at 1731808498570 2024-11-17T01:54:58,572 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:54:58,572 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T01:54:58,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T01:54:58,573 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T01:54:58,574 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T01:54:58,595 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(746): ClusterId : 1f97daa1-46f2-4024-a781-9f7075a910d8 2024-11-17T01:54:58,595 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(746): ClusterId : 1f97daa1-46f2-4024-a781-9f7075a910d8 2024-11-17T01:54:58,595 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T01:54:58,595 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T01:54:58,619 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T01:54:58,620 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T01:54:58,620 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T01:54:58,620 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T01:54:58,635 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T01:54:58,636 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T01:54:58,636 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764fe7dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T01:54:58,636 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10376ee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T01:54:58,648 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c6c0d7a0a7c0:34897 2024-11-17T01:54:58,648 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T01:54:58,648 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T01:54:58,648 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T01:54:58,649 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,38497,1731808497993 with port=34897, startcode=1731808498160 2024-11-17T01:54:58,649 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T01:54:58,651 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c6c0d7a0a7c0:39257 2024-11-17T01:54:58,651 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54407, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T01:54:58,651 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T01:54:58,652 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T01:54:58,652 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T01:54:58,652 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38497 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:54:58,652 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38497 {}] master.ServerManager(517): Registering regionserver=c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:54:58,652 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,38497,1731808497993 with port=39257, startcode=1731808498125 2024-11-17T01:54:58,653 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T01:54:58,654 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8 2024-11-17T01:54:58,654 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-17T01:54:58,654 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T01:54:58,655 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53917, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T01:54:58,655 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38497 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:54:58,655 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38497 {}] master.ServerManager(517): Registering regionserver=c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:54:58,657 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8 2024-11-17T01:54:58,657 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-17T01:54:58,657 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T01:54:58,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T01:54:58,721 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] zookeeper.ZKUtil(111): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:54:58,721 WARN [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T01:54:58,721 INFO [RS:1;c6c0d7a0a7c0:34897 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:54:58,721 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6c0d7a0a7c0,34897,1731808498160] 2024-11-17T01:54:58,721 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6c0d7a0a7c0,39257,1731808498125] 2024-11-17T01:54:58,721 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:54:58,722 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] zookeeper.ZKUtil(111): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:54:58,722 WARN [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T01:54:58,722 INFO [RS:0;c6c0d7a0a7c0:39257 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:54:58,722 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:54:58,724 WARN [c6c0d7a0a7c0:38497 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:54:58,731 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T01:54:58,732 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T01:54:58,734 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T01:54:58,734 INFO [RS:1;c6c0d7a0a7c0:34897 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T01:54:58,734 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,734 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T01:54:58,735 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T01:54:58,735 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,735 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,735 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,735 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T01:54:58,735 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,736 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,736 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,736 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T01:54:58,736 INFO [RS:0;c6c0d7a0a7c0:39257 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T01:54:58,736 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,736 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,736 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,736 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,736 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T01:54:58,736 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,736 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,736 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,736 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:54:58,736 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:54:58,736 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T01:54:58,737 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,737 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,737 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,737 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,737 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,737 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,737 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,737 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,737 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T01:54:58,737 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,737 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,737 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,738 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,738 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,738 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,34897,1731808498160-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T01:54:58,738 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,738 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,738 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,738 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:54:58,738 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:54:58,738 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:54:58,740 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,740 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,740 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,740 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,740 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,741 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,39257,1731808498125-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T01:54:58,752 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T01:54:58,752 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,34897,1731808498160-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,752 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,752 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.Replication(171): c6c0d7a0a7c0,34897,1731808498160 started 2024-11-17T01:54:58,755 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T01:54:58,755 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,39257,1731808498125-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,755 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,755 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.Replication(171): c6c0d7a0a7c0,39257,1731808498125 started 2024-11-17T01:54:58,766 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,766 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(1482): Serving as c6c0d7a0a7c0,34897,1731808498160, RpcServer on c6c0d7a0a7c0/172.17.0.2:34897, sessionid=0x10147ca23fd0002 2024-11-17T01:54:58,766 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T01:54:58,766 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:54:58,766 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,34897,1731808498160' 2024-11-17T01:54:58,766 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T01:54:58,767 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T01:54:58,767 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T01:54:58,767 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T01:54:58,767 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:54:58,768 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,34897,1731808498160' 2024-11-17T01:54:58,768 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T01:54:58,768 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T01:54:58,768 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T01:54:58,768 INFO [RS:1;c6c0d7a0a7c0:34897 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T01:54:58,768 INFO [RS:1;c6c0d7a0a7c0:34897 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T01:54:58,769 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:58,769 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(1482): Serving as c6c0d7a0a7c0,39257,1731808498125, RpcServer on c6c0d7a0a7c0/172.17.0.2:39257, sessionid=0x10147ca23fd0001 2024-11-17T01:54:58,769 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T01:54:58,769 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:54:58,769 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,39257,1731808498125' 2024-11-17T01:54:58,770 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T01:54:58,770 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T01:54:58,770 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T01:54:58,770 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T01:54:58,770 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:54:58,771 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,39257,1731808498125' 2024-11-17T01:54:58,771 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T01:54:58,771 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T01:54:58,771 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T01:54:58,772 INFO [RS:0;c6c0d7a0a7c0:39257 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T01:54:58,772 INFO [RS:0;c6c0d7a0a7c0:39257 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T01:54:58,869 INFO [RS:1;c6c0d7a0a7c0:34897 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T01:54:58,870 INFO [RS:1;c6c0d7a0a7c0:34897 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c6c0d7a0a7c0%2C34897%2C1731808498160, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,34897,1731808498160, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs, maxLogs=55 2024-11-17T01:54:58,873 INFO [RS:0;c6c0d7a0a7c0:39257 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T01:54:58,873 INFO [RS:0;c6c0d7a0a7c0:39257 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c6c0d7a0a7c0%2C39257%2C1731808498125, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,39257,1731808498125, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs, maxLogs=55 2024-11-17T01:54:58,890 INFO [RS:1;c6c0d7a0a7c0:34897 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,34897,1731808498160/c6c0d7a0a7c0%2C34897%2C1731808498160.1731808498872 2024-11-17T01:54:58,891 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T01:54:58,891 INFO [RS:0;c6c0d7a0a7c0:39257 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,39257,1731808498125/c6c0d7a0a7c0%2C39257%2C1731808498125.1731808498875 2024-11-17T01:54:58,894 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T01:54:58,975 DEBUG [c6c0d7a0a7c0:38497 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-17T01:54:58,975 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(204): Hosts are {c6c0d7a0a7c0=0} racks are {/default-rack=0} 2024-11-17T01:54:58,976 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T01:54:58,976 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T01:54:58,976 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T01:54:58,976 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T01:54:58,976 INFO [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T01:54:58,976 INFO [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T01:54:58,976 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T01:54:58,977 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:54:58,978 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,34897,1731808498160, state=OPENING 2024-11-17T01:54:59,001 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T01:54:59,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:59,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:59,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:59,010 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T01:54:59,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:59,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,34897,1731808498160}] 2024-11-17T01:54:59,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:59,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:59,164 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T01:54:59,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38585, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T01:54:59,173 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T01:54:59,173 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:54:59,174 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T01:54:59,175 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c6c0d7a0a7c0%2C34897%2C1731808498160.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,34897,1731808498160, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs, maxLogs=55 2024-11-17T01:54:59,192 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,34897,1731808498160/c6c0d7a0a7c0%2C34897%2C1731808498160.meta.1731808499176.meta 2024-11-17T01:54:59,192 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T01:54:59,193 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:54:59,193 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T01:54:59,193 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T01:54:59,193 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T01:54:59,194 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T01:54:59,194 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:59,194 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T01:54:59,194 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T01:54:59,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-17T01:54:59,196 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T01:54:59,196 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:59,196 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:59,196 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-17T01:54:59,197 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T01:54:59,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:59,197 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:59,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-17T01:54:59,198 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T01:54:59,198 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:59,199 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:59,199 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-17T01:54:59,199 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T01:54:59,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:59,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:54:59,200 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T01:54:59,201 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740 2024-11-17T01:54:59,201 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740 2024-11-17T01:54:59,202 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T01:54:59,202 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T01:54:59,202 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T01:54:59,203 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T01:54:59,203 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72596010, jitterRate=0.08176484704017639}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T01:54:59,203 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T01:54:59,204 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731808499194Writing region info on filesystem at 1731808499194Initializing all the Stores at 1731808499194Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808499194Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808499195 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808499195Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808499195Cleaning up temporary data from old regions at 1731808499202 (+7 ms)Running coprocessor post-open hooks at 1731808499203 (+1 ms)Region opened successfully at 1731808499204 (+1 ms) 2024-11-17T01:54:59,204 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731808499163 2024-11-17T01:54:59,207 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T01:54:59,207 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T01:54:59,207 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:54:59,208 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,34897,1731808498160, state=OPEN 2024-11-17T01:54:59,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:54:59,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:54:59,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:54:59,235 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:54:59,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:59,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:59,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:54:59,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T01:54:59,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,34897,1731808498160 in 225 msec 2024-11-17T01:54:59,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T01:54:59,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 665 msec 2024-11-17T01:54:59,240 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:54:59,240 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T01:54:59,245 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:59,245 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34897,1731808498160, seqNum=-1] 2024-11-17T01:54:59,245 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:59,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56751, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:59,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 745 msec 2024-11-17T01:54:59,254 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731808499254, completionTime=-1 2024-11-17T01:54:59,254 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-17T01:54:59,254 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T01:54:59,257 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T01:54:59,257 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731808559257 2024-11-17T01:54:59,257 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731808619257 2024-11-17T01:54:59,257 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-17T01:54:59,257 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,38497,1731808497993-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:59,257 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,38497,1731808497993-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:59,258 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,38497,1731808497993-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:59,258 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c6c0d7a0a7c0:38497, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:59,258 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:59,258 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:59,260 DEBUG [master/c6c0d7a0a7c0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T01:54:59,262 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.036sec 2024-11-17T01:54:59,263 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T01:54:59,263 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T01:54:59,263 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T01:54:59,263 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T01:54:59,263 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T01:54:59,263 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,38497,1731808497993-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T01:54:59,263 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,38497,1731808497993-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T01:54:59,264 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T01:54:59,265 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T01:54:59,265 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,38497,1731808497993-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:54:59,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:59,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,38497,-1 for getting cluster id 2024-11-17T01:54:59,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:59,297 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1f97daa1-46f2-4024-a781-9f7075a910d8' 2024-11-17T01:54:59,298 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:59,298 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1f97daa1-46f2-4024-a781-9f7075a910d8" 2024-11-17T01:54:59,298 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f0bec9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:59,298 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,38497,-1] 2024-11-17T01:54:59,299 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:59,299 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:59,302 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:59,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:59,304 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:59,306 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34897,1731808498160, seqNum=-1] 2024-11-17T01:54:59,306 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:59,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:59,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c6c0d7a0a7c0,38497,1731808497993 2024-11-17T01:54:59,311 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T01:54:59,312 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c6c0d7a0a7c0,38497,1731808497993 2024-11-17T01:54:59,312 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27fced3a 2024-11-17T01:54:59,312 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T01:54:59,314 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T01:54:59,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38497 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731808499310', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:54:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38497 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731808499310 2024-11-17T01:54:59,317 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731808499310 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:54:59,317 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:59,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38497 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731808499310" procId is: 4 2024-11-17T01:54:59,318 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731808499310 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:54:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:54:59,327 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731808499310-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a3a69be737d355f2260d6dce8947dca0, NAME => 'testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731808499310', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8 2024-11-17T01:54:59,334 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731808499310-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:59,334 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731808499310-pool-0 {}] regionserver.HRegion(1722): Closing a3a69be737d355f2260d6dce8947dca0, disabling compactions & flushes 2024-11-17T01:54:59,334 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731808499310-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:54:59,334 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731808499310-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:54:59,334 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731808499310-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. after waiting 0 ms 2024-11-17T01:54:59,334 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731808499310-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:54:59,334 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731808499310-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:54:59,334 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731808499310-pool-0 {}] regionserver.HRegion(1676): Region close journal for a3a69be737d355f2260d6dce8947dca0: Waiting for close lock at 1731808499334Disabling compacts and flushes for region at 1731808499334Disabling writes for close at 1731808499334Writing region close event to WAL at 1731808499334Closed at 1731808499334 2024-11-17T01:54:59,335 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731808499310 execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:54:59,335 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731808499335"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808499335"}]},"ts":"1731808499335"} 2024-11-17T01:54:59,338 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:54:59,339 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731808499310 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:54:59,339 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731808499310","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808499339"}]},"ts":"1731808499339"} 2024-11-17T01:54:59,341 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731808499310, state=ENABLING in hbase:meta 2024-11-17T01:54:59,342 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {c6c0d7a0a7c0=0} racks are {/default-rack=0} 2024-11-17T01:54:59,342 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T01:54:59,342 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T01:54:59,342 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T01:54:59,342 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T01:54:59,342 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T01:54:59,342 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T01:54:59,342 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T01:54:59,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731808499310, region=a3a69be737d355f2260d6dce8947dca0, ASSIGN}] 2024-11-17T01:54:59,344 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731808499310, region=a3a69be737d355f2260d6dce8947dca0, ASSIGN 2024-11-17T01:54:59,345 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731808499310, region=a3a69be737d355f2260d6dce8947dca0, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,34897,1731808498160; forceNewPlan=false, retain=false 2024-11-17T01:54:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:54:59,495 INFO [c6c0d7a0a7c0:38497 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-17T01:54:59,496 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a3a69be737d355f2260d6dce8947dca0, regionState=OPENING, regionLocation=c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:54:59,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731808499310, region=a3a69be737d355f2260d6dce8947dca0, ASSIGN because future has completed 2024-11-17T01:54:59,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3a69be737d355f2260d6dce8947dca0, server=c6c0d7a0a7c0,34897,1731808498160}] 2024-11-17T01:54:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:54:59,660 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:54:59,661 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a3a69be737d355f2260d6dce8947dca0, NAME => 'testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:54:59,662 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731808499310 a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:54:59,662 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:59,662 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:54:59,662 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:54:59,664 INFO [StoreOpener-a3a69be737d355f2260d6dce8947dca0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:54:59,668 INFO [StoreOpener-a3a69be737d355f2260d6dce8947dca0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3a69be737d355f2260d6dce8947dca0 columnFamilyName family 2024-11-17T01:54:59,669 DEBUG [StoreOpener-a3a69be737d355f2260d6dce8947dca0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:59,669 INFO [StoreOpener-a3a69be737d355f2260d6dce8947dca0-1 {}] regionserver.HStore(327): Store=a3a69be737d355f2260d6dce8947dca0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:59,669 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:54:59,670 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:54:59,670 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:54:59,671 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:54:59,671 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:54:59,672 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:54:59,680 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:54:59,681 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a3a69be737d355f2260d6dce8947dca0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69128490, jitterRate=0.030094772577285767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:54:59,681 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:54:59,682 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a3a69be737d355f2260d6dce8947dca0: Running coprocessor pre-open hook at 1731808499663Writing region info on filesystem at 1731808499663Initializing all the Stores at 1731808499663Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808499663Cleaning up temporary data from old regions at 1731808499671 (+8 ms)Running coprocessor post-open hooks at 1731808499681 (+10 ms)Region opened successfully at 1731808499682 (+1 ms) 2024-11-17T01:54:59,683 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0., pid=6, masterSystemTime=1731808499655 2024-11-17T01:54:59,687 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:54:59,687 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:54:59,687 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a3a69be737d355f2260d6dce8947dca0, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:54:59,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3a69be737d355f2260d6dce8947dca0, server=c6c0d7a0a7c0,34897,1731808498160 because future has completed 2024-11-17T01:54:59,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T01:54:59,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a3a69be737d355f2260d6dce8947dca0, server=c6c0d7a0a7c0,34897,1731808498160 in 191 msec 2024-11-17T01:54:59,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T01:54:59,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731808499310, region=a3a69be737d355f2260d6dce8947dca0, ASSIGN in 351 msec 2024-11-17T01:54:59,696 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731808499310 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:54:59,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731808499310","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808499696"}]},"ts":"1731808499696"} 2024-11-17T01:54:59,699 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731808499310, state=ENABLED in hbase:meta 2024-11-17T01:54:59,700 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731808499310 execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:54:59,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731808499310 in 385 msec 2024-11-17T01:54:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:54:59,950 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731808499310 completed 2024-11-17T01:54:59,950 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731808499310 get assigned. Timeout = 60000ms 2024-11-17T01:54:59,950 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:54:59,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731808499310 assigned to meta. Checking AM states. 2024-11-17T01:54:59,953 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:54:59,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731808499310 assigned. 2024-11-17T01:54:59,958 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731808499310', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0., hostname=c6c0d7a0a7c0,34897,1731808498160, seqNum=2] 2024-11-17T01:54:59,964 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a3a69be737d355f2260d6dce8947dca0 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-17T01:54:59,973 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bdc8d8f7057f41d48fa97fd3ae82f351 2024-11-17T01:54:59,986 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/.tmp/family/bdc8d8f7057f41d48fa97fd3ae82f351 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-17T01:54:59,986 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/.tmp/family/bdc8d8f7057f41d48fa97fd3ae82f351 2024-11-17T01:54:59,988 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/.tmp/family/bdc8d8f7057f41d48fa97fd3ae82f351 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/family/bdc8d8f7057f41d48fa97fd3ae82f351 2024-11-17T01:54:59,989 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/family/bdc8d8f7057f41d48fa97fd3ae82f351, delay=959 ms 2024-11-17T01:54:59,991 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/family/bdc8d8f7057f41d48fa97fd3ae82f351, entries=2, sequenceid=6, filesize=4.9 K 2024-11-17T01:54:59,991 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for a3a69be737d355f2260d6dce8947dca0 in 27ms, sequenceid=6, compaction requested=false 2024-11-17T01:54:59,991 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731808499310' 2024-11-17T01:54:59,992 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a3a69be737d355f2260d6dce8947dca0: 2024-11-17T01:55:00,159 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:55:00,951 DEBUG [hfile-prefetch-1731808497322 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key bdc8d8f7057f41d48fa97fd3ae82f351_0. 2024-11-17T01:55:00,953 DEBUG [hfile-prefetch-1731808497322 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key bdc8d8f7057f41d48fa97fd3ae82f351_117. 2024-11-17T01:55:00,954 DEBUG [hfile-prefetch-1731808497322 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/family/bdc8d8f7057f41d48fa97fd3ae82f351, with total blocks 2, and data blocks 1 2024-11-17T01:55:00,954 DEBUG [hfile-prefetch-1731808497322 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-17T01:55:00,954 DEBUG [hfile-prefetch-1731808497322 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/family/bdc8d8f7057f41d48fa97fd3ae82f351, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-17T01:55:00,954 DEBUG [hfile-prefetch-1731808497322 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/family/bdc8d8f7057f41d48fa97fd3ae82f351. Will sleep 100ms and try the verification again. 2024-11-17T01:55:00,992 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;c6c0d7a0a7c0:34897,5,FailOnTimeoutGroup] 2024-11-17T01:55:00,992 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6c0d7a0a7c0,34897,1731808498160' ***** 2024-11-17T01:55:00,992 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-17T01:55:00,993 INFO [RS:1;c6c0d7a0a7c0:34897 {}] bucket.BucketCache(1738): Disabling cache 2024-11-17T01:55:01,040 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T01:55:01,040 INFO [regionserver/c6c0d7a0a7c0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:55:01,046 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-17T01:55:01,046 INFO [RS:1;c6c0d7a0a7c0:34897 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.persistence 2024-11-17T01:55:01,046 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-17T01:55:01,046 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-17T01:55:01,047 INFO [RS:1;c6c0d7a0a7c0:34897 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-17T01:55:01,047 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-17T01:55:01,055 DEBUG [hfile-prefetch-1731808497322 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/family/bdc8d8f7057f41d48fa97fd3ae82f351, with total blocks 2, and data blocks 1 2024-11-17T01:55:01,055 DEBUG [hfile-prefetch-1731808497322 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-17T01:55:01,055 DEBUG [hfile-prefetch-1731808497322 {}] bucket.BucketCache(2293): found block for file bdc8d8f7057f41d48fa97fd3ae82f351 in the backing map. Acquiring read lock for offset 0 2024-11-17T01:55:01,055 DEBUG [hfile-prefetch-1731808497322 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/family/bdc8d8f7057f41d48fa97fd3ae82f351 has now been fully cached. 2024-11-17T01:55:01,056 DEBUG [hfile-prefetch-1731808497322 {}] hfile.PrefetchExecutor(121): Prefetch completed for bdc8d8f7057f41d48fa97fd3ae82f351 2024-11-17T01:55:01,057 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] bucket.BucketCache(1487): Thread RS:1;c6c0d7a0a7c0:34897 started persisting bucket cache to file 2024-11-17T01:55:01,057 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T01:55:01,057 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-17T01:55:01,064 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-17T01:55:01,065 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731808501057 2024-11-17T01:55:01,065 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] bucket.BucketCache(1509): Thread RS:1;c6c0d7a0a7c0:34897 finished persisting bucket cache to file, renaming 2024-11-17T01:55:01,065 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T01:55:01,065 INFO [RS:1;c6c0d7a0a7c0:34897 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T01:55:01,065 INFO [RS:1;c6c0d7a0a7c0:34897 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T01:55:01,065 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:55:01,066 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(959): stopping server c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:55:01,066 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T01:55:01,066 INFO [RS:1;c6c0d7a0a7c0:34897 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c6c0d7a0a7c0:34897. 2024-11-17T01:55:01,066 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-17T01:55:01,066 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a3a69be737d355f2260d6dce8947dca0, disabling compactions & flushes 2024-11-17T01:55:01,066 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:55:01,066 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:55:01,066 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:55:01,066 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:55:01,066 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T01:55:01,066 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. after waiting 0 ms 2024-11-17T01:55:01,066 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T01:55:01,066 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:55:01,066 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T01:55:01,067 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T01:55:01,067 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-17T01:55:01,067 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(1325): Online Regions={a3a69be737d355f2260d6dce8947dca0=testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0., 1588230740=hbase:meta,,1.1588230740} 2024-11-17T01:55:01,067 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, a3a69be737d355f2260d6dce8947dca0 2024-11-17T01:55:01,067 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-17T01:55:01,067 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T01:55:01,067 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T01:55:01,068 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T01:55:01,068 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T01:55:01,068 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T01:55:01,068 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-17T01:55:01,073 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/default/testBlockEvictionOnGracefulStop1731808499310/a3a69be737d355f2260d6dce8947dca0/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-17T01:55:01,073 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:55:01,073 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a3a69be737d355f2260d6dce8947dca0: Waiting for close lock at 1731808501066Running coprocessor pre-close hooks at 1731808501066Disabling compacts and flushes for region at 1731808501066Disabling writes for close at 1731808501066Writing region close event to WAL at 1731808501068 (+2 ms)Running coprocessor post-close hooks at 1731808501073 (+5 ms)Closed at 1731808501073 2024-11-17T01:55:01,073 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0. 2024-11-17T01:55:01,077 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on daf761acceb6459fbe6a19da179c5701 2024-11-17T01:55:01,089 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/info/daf761acceb6459fbe6a19da179c5701 is 209, key is testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0./info:regioninfo/1731808499687/Put/seqid=0 2024-11-17T01:55:01,090 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/info/daf761acceb6459fbe6a19da179c5701 2024-11-17T01:55:01,097 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 006f1842c11d49f89d85be3e93685973 2024-11-17T01:55:01,109 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/ns/006f1842c11d49f89d85be3e93685973 is 43, key is default/ns:d/1731808499248/Put/seqid=0 2024-11-17T01:55:01,109 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/ns/006f1842c11d49f89d85be3e93685973 2024-11-17T01:55:01,116 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 187dc8d8529a476fa9be957e54db48fc 2024-11-17T01:55:01,128 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/table/187dc8d8529a476fa9be957e54db48fc is 80, key is testBlockEvictionOnGracefulStop1731808499310/table:state/1731808499696/Put/seqid=0 2024-11-17T01:55:01,129 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/table/187dc8d8529a476fa9be957e54db48fc 2024-11-17T01:55:01,132 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/info/daf761acceb6459fbe6a19da179c5701 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/daf761acceb6459fbe6a19da179c5701 2024-11-17T01:55:01,134 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/daf761acceb6459fbe6a19da179c5701, delay=981 ms 2024-11-17T01:55:01,135 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/daf761acceb6459fbe6a19da179c5701, entries=10, sequenceid=11, filesize=7.0 K 2024-11-17T01:55:01,135 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/ns/006f1842c11d49f89d85be3e93685973 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/ns/006f1842c11d49f89d85be3e93685973 2024-11-17T01:55:01,137 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/ns/006f1842c11d49f89d85be3e93685973, delay=918 ms 2024-11-17T01:55:01,138 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/ns/006f1842c11d49f89d85be3e93685973, entries=2, sequenceid=11, filesize=5.0 K 2024-11-17T01:55:01,139 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/table/187dc8d8529a476fa9be957e54db48fc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/table/187dc8d8529a476fa9be957e54db48fc 2024-11-17T01:55:01,139 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-17T01:55:01,139 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-17T01:55:01,139 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T01:55:01,139 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T01:55:01,140 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/table/187dc8d8529a476fa9be957e54db48fc, delay=901 ms 2024-11-17T01:55:01,141 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/table/187dc8d8529a476fa9be957e54db48fc, entries=2, sequenceid=11, filesize=5.3 K 2024-11-17T01:55:01,142 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T01:55:01,142 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731808501139 2024-11-17T01:55:01,142 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-17T01:55:01,142 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=11, compaction requested=false 2024-11-17T01:55:01,143 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/daf761acceb6459fbe6a19da179c5701 2024-11-17T01:55:01,143 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/ns/006f1842c11d49f89d85be3e93685973 2024-11-17T01:55:01,144 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/table/187dc8d8529a476fa9be957e54db48fc 2024-11-17T01:55:01,148 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-17T01:55:01,148 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T01:55:01,148 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T01:55:01,148 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731808501067Running coprocessor pre-close hooks at 1731808501067Disabling compacts and flushes for region at 1731808501067Disabling writes for close at 1731808501068 (+1 ms)Obtaining lock to block concurrent updates at 1731808501068Preparing flush snapshotting stores in 1588230740 at 1731808501068Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731808501069 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731808501069Flushing 1588230740/info: creating writer at 1731808501069Flushing 1588230740/info: appending metadata at 1731808501089 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731808501089Flushing 1588230740/ns: creating writer at 1731808501092 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731808501108 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731808501108Flushing 1588230740/table: creating writer at 1731808501111 (+3 ms)Flushing 1588230740/table: appending metadata at 1731808501128 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731808501128Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f9591b9: reopening flushed file at 1731808501132 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30f962f9: reopening flushed file at 1731808501135 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2224db88: reopening flushed file at 1731808501138 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=11, compaction requested=false at 1731808501142 (+4 ms)Writing region close event to WAL at 1731808501144 (+2 ms)Running coprocessor post-close hooks at 1731808501148 (+4 ms)Closed at 1731808501148 2024-11-17T01:55:01,149 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T01:55:01,267 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(976): stopping server c6c0d7a0a7c0,34897,1731808498160; all regions closed. 2024-11-17T01:55:01,269 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs 2024-11-17T01:55:01,269 INFO [RS:1;c6c0d7a0a7c0:34897 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C34897%2C1731808498160.meta:.meta(num 1731808499176) 2024-11-17T01:55:01,271 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs 2024-11-17T01:55:01,271 INFO [RS:1;c6c0d7a0a7c0:34897 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C34897%2C1731808498160:(num 1731808498872) 2024-11-17T01:55:01,271 DEBUG [RS:1;c6c0d7a0a7c0:34897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:55:01,271 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:55:01,271 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T01:55:01,271 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.ChoreService(370): Chore service for: regionserver/c6c0d7a0a7c0:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T01:55:01,271 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T01:55:01,271 INFO [regionserver/c6c0d7a0a7c0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T01:55:01,272 INFO [RS:1;c6c0d7a0a7c0:34897 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34897 2024-11-17T01:55:01,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6c0d7a0a7c0,34897,1731808498160 2024-11-17T01:55:01,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T01:55:01,309 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T01:55:01,310 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6c0d7a0a7c0,34897,1731808498160] 2024-11-17T01:55:01,326 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6c0d7a0a7c0,34897,1731808498160 already deleted, retry=false 2024-11-17T01:55:01,326 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of c6c0d7a0a7c0,34897,1731808498160 on c6c0d7a0a7c0,38497,1731808497993 2024-11-17T01:55:01,331 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure c6c0d7a0a7c0,34897,1731808498160, splitWal=true, meta=true 2024-11-17T01:55:01,333 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for c6c0d7a0a7c0,34897,1731808498160 (carryingMeta=true) c6c0d7a0a7c0,34897,1731808498160/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2fec16f4[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-17T01:55:01,333 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure c6c0d7a0a7c0,34897,1731808498160, splitWal=true, meta=true 2024-11-17T01:55:01,334 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure c6c0d7a0a7c0,34897,1731808498160, splitWal=true, meta=true, isMeta: true 2024-11-17T01:55:01,334 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,34897,1731808498160-splitting 2024-11-17T01:55:01,335 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,34897,1731808498160-splitting dir is empty, no logs to split. 2024-11-17T01:55:01,335 INFO [PEWorker-2 {}] master.SplitWALManager(105): c6c0d7a0a7c0,34897,1731808498160 WAL count=0, meta=true 2024-11-17T01:55:01,336 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,34897,1731808498160-splitting dir is empty, no logs to split. 2024-11-17T01:55:01,336 INFO [PEWorker-2 {}] master.SplitWALManager(105): c6c0d7a0a7c0,34897,1731808498160 WAL count=0, meta=true 2024-11-17T01:55:01,337 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if c6c0d7a0a7c0,34897,1731808498160 WAL splitting is done? wals=0, meta=true 2024-11-17T01:55:01,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T01:55:01,339 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T01:55:01,340 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T01:55:01,418 INFO [RS:1;c6c0d7a0a7c0:34897 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T01:55:01,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:55:01,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34897-0x10147ca23fd0002, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:55:01,418 INFO [RS:1;c6c0d7a0a7c0:34897 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6c0d7a0a7c0,34897,1731808498160; zookeeper connection closed. 2024-11-17T01:55:01,418 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4994e58c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4994e58c 2024-11-17T01:55:01,490 DEBUG [c6c0d7a0a7c0:38497 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-17T01:55:01,490 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:55:01,491 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,39257,1731808498125, state=OPENING 2024-11-17T01:55:01,509 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T01:55:01,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:55:01,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T01:55:01,509 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T01:55:01,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:55:01,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T01:55:01,509 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T01:55:01,509 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T01:55:01,510 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37061 2024-11-17T01:55:01,511 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37061 connecting to ZooKeeper ensemble=127.0.0.1:49311 2024-11-17T01:55:01,511 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:55:01,511 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:55:01,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:55:01,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:55:01,543 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T01:55:01,543 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:55:01,543 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:55:01,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,39257,1731808498125}] 2024-11-17T01:55:01,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370610x0, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T01:55:01,552 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37061-0x10147ca23fd0003, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-17T01:55:01,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37061-0x10147ca23fd0003 connected 2024-11-17T01:55:01,552 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-17T01:55:01,553 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T01:55:01,554 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.cache 2024-11-17T01:55:01,554 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-17T01:55:01,554 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-17T01:55:01,554 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-17T01:55:01,555 INFO [Thread-208 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-17T01:55:01,555 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-17T01:55:01,555 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T01:55:01,556 INFO [Thread-208 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-17T01:55:01,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37061-0x10147ca23fd0003, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T01:55:01,557 INFO [Thread-208 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-17T01:55:01,559 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37061-0x10147ca23fd0003, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T01:55:01,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37061 2024-11-17T01:55:01,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37061 2024-11-17T01:55:01,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37061 2024-11-17T01:55:01,561 DEBUG [Thread-208 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-17T01:55:01,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37061 2024-11-17T01:55:01,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37061 2024-11-17T01:55:01,562 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-17T01:55:01,562 INFO [Thread-208 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-17T01:55:01,562 INFO [Thread-208 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.persistence is complete. 2024-11-17T01:55:01,564 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer(746): ClusterId : 1f97daa1-46f2-4024-a781-9f7075a910d8 2024-11-17T01:55:01,564 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T01:55:01,569 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T01:55:01,569 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T01:55:01,577 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T01:55:01,577 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@171af37c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T01:55:01,588 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;c6c0d7a0a7c0:37061 2024-11-17T01:55:01,588 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T01:55:01,588 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T01:55:01,588 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T01:55:01,589 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,38497,1731808497993 with port=37061, startcode=1731808501508 2024-11-17T01:55:01,589 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T01:55:01,591 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34179, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T01:55:01,592 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38497 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6c0d7a0a7c0,37061,1731808501508 2024-11-17T01:55:01,592 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38497 {}] master.ServerManager(517): Registering regionserver=c6c0d7a0a7c0,37061,1731808501508 2024-11-17T01:55:01,594 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8 2024-11-17T01:55:01,594 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-17T01:55:01,594 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T01:55:01,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T01:55:01,601 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] zookeeper.ZKUtil(111): regionserver:37061-0x10147ca23fd0003, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6c0d7a0a7c0,37061,1731808501508 2024-11-17T01:55:01,602 WARN [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T01:55:01,602 INFO [RS:2;c6c0d7a0a7c0:37061 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:55:01,602 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6c0d7a0a7c0,37061,1731808501508] 2024-11-17T01:55:01,602 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,37061,1731808501508 2024-11-17T01:55:01,607 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T01:55:01,610 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T01:55:01,610 INFO [RS:2;c6c0d7a0a7c0:37061 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T01:55:01,610 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:01,611 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T01:55:01,611 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T01:55:01,611 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:01,611 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:01,611 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:01,611 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:01,612 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:01,612 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:01,612 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T01:55:01,612 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:01,612 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:01,612 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:01,612 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:01,612 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:01,612 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:01,612 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:55:01,612 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:55:01,613 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:01,613 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:01,613 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:01,613 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:01,613 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:01,613 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,37061,1731808501508-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T01:55:01,627 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T01:55:01,628 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,37061,1731808501508-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:01,628 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:01,628 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.Replication(171): c6c0d7a0a7c0,37061,1731808501508 started 2024-11-17T01:55:01,642 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:01,642 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer(1482): Serving as c6c0d7a0a7c0,37061,1731808501508, RpcServer on c6c0d7a0a7c0/172.17.0.2:37061, sessionid=0x10147ca23fd0003 2024-11-17T01:55:01,642 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T01:55:01,642 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6c0d7a0a7c0,37061,1731808501508 2024-11-17T01:55:01,643 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,37061,1731808501508' 2024-11-17T01:55:01,643 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T01:55:01,643 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T01:55:01,644 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T01:55:01,644 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T01:55:01,644 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6c0d7a0a7c0,37061,1731808501508 2024-11-17T01:55:01,644 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,37061,1731808501508' 2024-11-17T01:55:01,644 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T01:55:01,644 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T01:55:01,645 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T01:55:01,645 INFO [RS:2;c6c0d7a0a7c0:37061 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T01:55:01,645 INFO [RS:2;c6c0d7a0a7c0:37061 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T01:55:01,697 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T01:55:01,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57723, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T01:55:01,701 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T01:55:01,701 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:55:01,701 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T01:55:01,702 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c6c0d7a0a7c0%2C39257%2C1731808498125.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,39257,1731808498125, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs, maxLogs=55 2024-11-17T01:55:01,726 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,39257,1731808498125/c6c0d7a0a7c0%2C39257%2C1731808498125.meta.1731808501703.meta 2024-11-17T01:55:01,726 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T01:55:01,726 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:55:01,726 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T01:55:01,727 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T01:55:01,727 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T01:55:01,727 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T01:55:01,727 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:55:01,727 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T01:55:01,727 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T01:55:01,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-17T01:55:01,729 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T01:55:01,729 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:01,730 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/daf761acceb6459fbe6a19da179c5701, delay=971 ms 2024-11-17T01:55:01,731 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/daf761acceb6459fbe6a19da179c5701 2024-11-17T01:55:01,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:01,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-17T01:55:01,732 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T01:55:01,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:01,733 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/ns/006f1842c11d49f89d85be3e93685973, delay=922 ms 2024-11-17T01:55:01,734 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/ns/006f1842c11d49f89d85be3e93685973 2024-11-17T01:55:01,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:01,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-17T01:55:01,735 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T01:55:01,735 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:01,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:01,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-17T01:55:01,736 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T01:55:01,736 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:01,738 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/table/187dc8d8529a476fa9be957e54db48fc, delay=998 ms 2024-11-17T01:55:01,739 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/table/187dc8d8529a476fa9be957e54db48fc 2024-11-17T01:55:01,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:01,739 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T01:55:01,740 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740 2024-11-17T01:55:01,741 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740 2024-11-17T01:55:01,741 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T01:55:01,741 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T01:55:01,742 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T01:55:01,743 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T01:55:01,744 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60578253, jitterRate=-0.09731368720531464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T01:55:01,744 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T01:55:01,745 INFO [RS:2;c6c0d7a0a7c0:37061 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T01:55:01,746 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731808501727Writing region info on filesystem at 1731808501727Initializing all the Stores at 1731808501727Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808501728 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808501728Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808501728Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808501728Cleaning up temporary data from old regions at 1731808501741 (+13 ms)Running coprocessor post-open hooks at 1731808501744 (+3 ms)Region opened successfully at 1731808501746 (+2 ms) 2024-11-17T01:55:01,746 INFO [RS:2;c6c0d7a0a7c0:37061 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=c6c0d7a0a7c0%2C37061%2C1731808501508, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,37061,1731808501508, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs, maxLogs=55 2024-11-17T01:55:01,747 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731808501696 2024-11-17T01:55:01,750 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T01:55:01,750 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T01:55:01,750 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:55:01,751 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,39257,1731808498125, state=OPEN 2024-11-17T01:55:01,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37061-0x10147ca23fd0003, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:55:01,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:55:01,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:55:01,759 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:55:01,759 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:55:01,759 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:55:01,759 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:55:01,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-17T01:55:01,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,39257,1731808498125 in 216 msec 2024-11-17T01:55:01,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T01:55:01,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 425 msec 2024-11-17T01:55:01,766 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): c6c0d7a0a7c0,34897,1731808498160 had 2 regions 2024-11-17T01:55:01,767 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure c6c0d7a0a7c0,34897,1731808498160, splitWal=true, meta=true, isMeta: false 2024-11-17T01:55:01,768 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,34897,1731808498160-splitting dir is empty, no logs to split. 2024-11-17T01:55:01,768 INFO [PEWorker-2 {}] master.SplitWALManager(105): c6c0d7a0a7c0,34897,1731808498160 WAL count=0, meta=false 2024-11-17T01:55:01,769 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,34897,1731808498160-splitting dir is empty, no logs to split. 2024-11-17T01:55:01,769 INFO [PEWorker-2 {}] master.SplitWALManager(105): c6c0d7a0a7c0,34897,1731808498160 WAL count=0, meta=false 2024-11-17T01:55:01,769 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if c6c0d7a0a7c0,34897,1731808498160 WAL splitting is done? wals=0, meta=false 2024-11-17T01:55:01,771 INFO [RS:2;c6c0d7a0a7c0:37061 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/WALs/c6c0d7a0a7c0,37061,1731808501508/c6c0d7a0a7c0%2C37061%2C1731808501508.1731808501747 2024-11-17T01:55:01,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731808499310, region=a3a69be737d355f2260d6dce8947dca0, ASSIGN}] 2024-11-17T01:55:01,772 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731808499310, region=a3a69be737d355f2260d6dce8947dca0, ASSIGN 2024-11-17T01:55:01,773 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731808499310, region=a3a69be737d355f2260d6dce8947dca0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T01:55:01,774 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T01:55:01,924 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(204): Hosts are {c6c0d7a0a7c0=0} racks are {/default-rack=0} 2024-11-17T01:55:01,924 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T01:55:01,924 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T01:55:01,924 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T01:55:01,924 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T01:55:01,924 INFO [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T01:55:01,924 INFO [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T01:55:01,925 DEBUG [c6c0d7a0a7c0:38497 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T01:55:01,925 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a3a69be737d355f2260d6dce8947dca0, regionState=OPENING, regionLocation=c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:55:01,927 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server c6c0d7a0a7c0:34897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:55:01,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34897,1731808498160, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34897,1731808498160, seqNum=-1, error=java.net.ConnectException: Call to address=c6c0d7a0a7c0:34897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34897 2024-11-17T01:55:01,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34897,1731808498160, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-17T01:55:01,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34897,1731808498160, seqNum=-1 from cache 2024-11-17T01:55:01,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address c6c0d7a0a7c0:34897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34897 2024-11-17T01:55:02,039 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:55:02,039 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,39257,1731808498125, seqNum=-1] 2024-11-17T01:55:02,040 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:55:02,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50361, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:55:02,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731808499310, region=a3a69be737d355f2260d6dce8947dca0, ASSIGN because future has completed 2024-11-17T01:55:02,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3a69be737d355f2260d6dce8947dca0, server=c6c0d7a0a7c0,39257,1731808498125}] 2024-11-17T01:55:02,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T01:55:02,143 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:55:02,143 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:55:02,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:55:02,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:55:02,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:55:02,143 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T01:55:02,144 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1442517092, stopped=false 2024-11-17T01:55:02,144 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c6c0d7a0a7c0,38497,1731808497993 2024-11-17T01:55:02,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37061-0x10147ca23fd0003, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T01:55:02,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T01:55:02,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:02,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T01:55:02,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:02,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37061-0x10147ca23fd0003, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:02,177 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T01:55:02,177 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:55:02,177 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:55:02,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:55:02,178 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6c0d7a0a7c0,39257,1731808498125' ***** 2024-11-17T01:55:02,178 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T01:55:02,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:55:02,178 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6c0d7a0a7c0,37061,1731808501508' ***** 2024-11-17T01:55:02,178 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T01:55:02,178 INFO [RS:0;c6c0d7a0a7c0:39257 {}] bucket.BucketCache(1738): Disabling cache 2024-11-17T01:55:02,178 INFO [RS:0;c6c0d7a0a7c0:39257 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.persistence 2024-11-17T01:55:02,178 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-17T01:55:02,178 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-17T01:55:02,178 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-17T01:55:02,178 INFO [RS:0;c6c0d7a0a7c0:39257 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-17T01:55:02,178 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-17T01:55:02,179 INFO [RS:2;c6c0d7a0a7c0:37061 {}] bucket.BucketCache(1738): Disabling cache 2024-11-17T01:55:02,180 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37061-0x10147ca23fd0003, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:55:02,180 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:55:02,181 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-17T01:55:02,181 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-17T01:55:02,180 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-17T01:55:02,180 INFO [RS:2;c6c0d7a0a7c0:37061 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f78fe0d-6324-808f-d52e-c20350d60af9/bucket.persistence 2024-11-17T01:55:02,181 INFO [RS:2;c6c0d7a0a7c0:37061 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-17T01:55:02,181 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-17T01:55:02,189 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] bucket.BucketCache(1487): Thread RS:0;c6c0d7a0a7c0:39257 started persisting bucket cache to file 2024-11-17T01:55:02,189 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T01:55:02,189 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T01:55:02,191 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] bucket.BucketCache(1487): Thread RS:2;c6c0d7a0a7c0:37061 started persisting bucket cache to file 2024-11-17T01:55:02,192 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T01:55:02,192 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T01:55:02,192 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T01:55:02,192 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731808502189 2024-11-17T01:55:02,193 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] bucket.BucketCache(1509): Thread RS:0;c6c0d7a0a7c0:39257 finished persisting bucket cache to file, renaming 2024-11-17T01:55:02,193 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T01:55:02,193 INFO [RS:0;c6c0d7a0a7c0:39257 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T01:55:02,193 INFO [RS:0;c6c0d7a0a7c0:39257 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T01:55:02,193 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(959): stopping server c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:55:02,193 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T01:55:02,193 INFO [RS:0;c6c0d7a0a7c0:39257 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c6c0d7a0a7c0:39257. 2024-11-17T01:55:02,193 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:55:02,193 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:55:02,194 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T01:55:02,194 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T01:55:02,194 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T01:55:02,194 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T01:55:02,194 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T01:55:02,196 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T01:55:02,196 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T01:55:02,196 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T01:55:02,196 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-17T01:55:02,197 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T01:55:02,197 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T01:55:02,197 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T01:55:02,197 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731808502192 2024-11-17T01:55:02,197 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T01:55:02,197 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] bucket.BucketCache(1509): Thread RS:2;c6c0d7a0a7c0:37061 finished persisting bucket cache to file, renaming 2024-11-17T01:55:02,197 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T01:55:02,197 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T01:55:02,197 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=485 B heapSize=1.70 KB 2024-11-17T01:55:02,197 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T01:55:02,197 INFO [RS:2;c6c0d7a0a7c0:37061 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T01:55:02,197 INFO [RS:2;c6c0d7a0a7c0:37061 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T01:55:02,197 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer(959): stopping server c6c0d7a0a7c0,37061,1731808501508 2024-11-17T01:55:02,197 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T01:55:02,198 INFO [RS:2;c6c0d7a0a7c0:37061 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;c6c0d7a0a7c0:37061. 2024-11-17T01:55:02,198 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:55:02,198 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:55:02,198 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer(976): stopping server c6c0d7a0a7c0,37061,1731808501508; all regions closed. 2024-11-17T01:55:02,197 DEBUG [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(271): Request to c6c0d7a0a7c0,39257,1731808498125 failed, try=0 org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c6c0d7a0a7c0:39257 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$Stub.executeProcedures(AdminProtos.java:35059) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.lambda$executeProcedures$20(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:103) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:121) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.executeProcedures(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 11 more 2024-11-17T01:55:02,198 WARN [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): c6c0d7a0a7c0,39257,1731808498125 tells us DoNotRetry due to org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c6c0d7a0a7c0:39257 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException, try=0, give up 2024-11-17T01:55:02,199 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T01:55:02,199 WARN [RSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(139): The remote operation pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3a69be737d355f2260d6dce8947dca0, server=c6c0d7a0a7c0,39257,1731808498125 for region state=OPENING, location=c6c0d7a0a7c0,39257,1731808498125, table=testBlockEvictionOnGracefulStop1731808499310, region=a3a69be737d355f2260d6dce8947dca0 to server c6c0d7a0a7c0,39257,1731808498125 failed org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c6c0d7a0a7c0:39257 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$Stub.executeProcedures(AdminProtos.java:35059) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.lambda$executeProcedures$20(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:103) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:121) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.executeProcedures(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 11 more 2024-11-17T01:55:02,206 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs 2024-11-17T01:55:02,206 INFO [RS:2;c6c0d7a0a7c0:37061 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C37061%2C1731808501508:(num 1731808501747) 2024-11-17T01:55:02,206 DEBUG [RS:2;c6c0d7a0a7c0:37061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:55:02,206 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:55:02,206 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T01:55:02,206 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.ChoreService(370): Chore service for: regionserver/c6c0d7a0a7c0:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-17T01:55:02,207 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T01:55:02,207 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T01:55:02,207 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T01:55:02,207 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T01:55:02,207 INFO [RS:2;c6c0d7a0a7c0:37061 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37061 2024-11-17T01:55:02,207 INFO [regionserver/c6c0d7a0a7c0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T01:55:02,214 INFO [regionserver/c6c0d7a0a7c0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:55:02,214 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T01:55:02,215 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a751688ef9c74c64a54a642550b4080c 2024-11-17T01:55:02,215 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T01:55:02,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T01:55:02,217 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T01:55:02,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T01:55:02,218 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T01:55:02,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T01:55:02,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37061-0x10147ca23fd0003, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6c0d7a0a7c0,37061,1731808501508 2024-11-17T01:55:02,226 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T01:55:02,227 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6c0d7a0a7c0,37061,1731808501508] 2024-11-17T01:55:02,228 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/info/a751688ef9c74c64a54a642550b4080c is 209, key is testBlockEvictionOnGracefulStop1731808499310,,1731808499315.a3a69be737d355f2260d6dce8947dca0./info:regioninfo/1731808501925/Put/seqid=0 2024-11-17T01:55:02,229 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=485 B at sequenceid=18 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/info/a751688ef9c74c64a54a642550b4080c 2024-11-17T01:55:02,232 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/.tmp/info/a751688ef9c74c64a54a642550b4080c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/a751688ef9c74c64a54a642550b4080c 2024-11-17T01:55:02,233 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/a751688ef9c74c64a54a642550b4080c, delay=979 ms 2024-11-17T01:55:02,234 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/a751688ef9c74c64a54a642550b4080c, entries=3, sequenceid=18, filesize=5.9 K 2024-11-17T01:55:02,235 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 38ms, sequenceid=18, compaction requested=false 2024-11-17T01:55:02,236 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/daf761acceb6459fbe6a19da179c5701 2024-11-17T01:55:02,237 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/info/a751688ef9c74c64a54a642550b4080c 2024-11-17T01:55:02,237 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/ns/006f1842c11d49f89d85be3e93685973 2024-11-17T01:55:02,238 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/table/187dc8d8529a476fa9be957e54db48fc 2024-11-17T01:55:02,242 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/data/hbase/meta/1588230740/recovered.edits/21.seqid, newMaxSeqId=21, maxSeqId=14 2024-11-17T01:55:02,242 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6c0d7a0a7c0,37061,1731808501508 already deleted, retry=false 2024-11-17T01:55:02,242 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T01:55:02,242 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T01:55:02,243 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731808502196Running coprocessor pre-close hooks at 1731808502196Disabling compacts and flushes for region at 1731808502196Disabling writes for close at 1731808502197 (+1 ms)Obtaining lock to block concurrent updates at 1731808502197Preparing flush snapshotting stores in 1588230740 at 1731808502197Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=485, getHeapSize=1672, getOffHeapSize=0, getCellsCount=3 at 1731808502198 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731808502200 (+2 ms)Flushing 1588230740/info: creating writer at 1731808502200Flushing 1588230740/info: appending metadata at 1731808502228 (+28 ms)Flushing 1588230740/info: closing flushed file at 1731808502228Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a26a9ce: reopening flushed file at 1731808502231 (+3 ms)Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 38ms, sequenceid=18, compaction requested=false at 1731808502235 (+4 ms)Writing region close event to WAL at 1731808502238 (+3 ms)Running coprocessor post-close hooks at 1731808502242 (+4 ms)Closed at 1731808502242 2024-11-17T01:55:02,243 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c6c0d7a0a7c0,37061,1731808501508 expired; onlineServers=1 2024-11-17T01:55:02,243 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T01:55:02,244 INFO [regionserver/c6c0d7a0a7c0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:55:02,335 INFO [RS:2;c6c0d7a0a7c0:37061 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T01:55:02,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37061-0x10147ca23fd0003, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:55:02,335 INFO [RS:2;c6c0d7a0a7c0:37061 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6c0d7a0a7c0,37061,1731808501508; zookeeper connection closed. 2024-11-17T01:55:02,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37061-0x10147ca23fd0003, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:55:02,335 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5d8bc444 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5d8bc444 2024-11-17T01:55:02,396 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(976): stopping server c6c0d7a0a7c0,39257,1731808498125; all regions closed. 2024-11-17T01:55:02,399 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs 2024-11-17T01:55:02,399 INFO [RS:0;c6c0d7a0a7c0:39257 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C39257%2C1731808498125.meta:.meta(num 1731808501703) 2024-11-17T01:55:02,401 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/oldWALs 2024-11-17T01:55:02,401 INFO [RS:0;c6c0d7a0a7c0:39257 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C39257%2C1731808498125:(num 1731808498875) 2024-11-17T01:55:02,402 DEBUG [RS:0;c6c0d7a0a7c0:39257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:55:02,402 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:55:02,402 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T01:55:02,402 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.ChoreService(370): Chore service for: regionserver/c6c0d7a0a7c0:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T01:55:02,402 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T01:55:02,402 INFO [regionserver/c6c0d7a0a7c0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T01:55:02,403 INFO [RS:0;c6c0d7a0a7c0:39257 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39257 2024-11-17T01:55:02,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T01:55:02,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6c0d7a0a7c0,39257,1731808498125 2024-11-17T01:55:02,434 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T01:55:02,435 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6c0d7a0a7c0,39257,1731808498125] 2024-11-17T01:55:02,451 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6c0d7a0a7c0,39257,1731808498125 already deleted, retry=false 2024-11-17T01:55:02,451 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c6c0d7a0a7c0,39257,1731808498125 expired; onlineServers=0 2024-11-17T01:55:02,451 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c6c0d7a0a7c0,38497,1731808497993' ***** 2024-11-17T01:55:02,451 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T01:55:02,451 INFO [M:0;c6c0d7a0a7c0:38497 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T01:55:02,451 INFO [M:0;c6c0d7a0a7c0:38497 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T01:55:02,451 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T01:55:02,451 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T01:55:02,451 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T01:55:02,451 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808498527 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808498527,5,FailOnTimeoutGroup] 2024-11-17T01:55:02,451 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808498529 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808498529,5,FailOnTimeoutGroup] 2024-11-17T01:55:02,452 INFO [M:0;c6c0d7a0a7c0:38497 {}] hbase.ChoreService(370): Chore service for: master/c6c0d7a0a7c0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T01:55:02,452 INFO [M:0;c6c0d7a0a7c0:38497 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T01:55:02,452 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] master.HMaster(1795): Stopping service threads 2024-11-17T01:55:02,452 INFO [M:0;c6c0d7a0a7c0:38497 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T01:55:02,452 INFO [M:0;c6c0d7a0a7c0:38497 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T01:55:02,452 INFO [M:0;c6c0d7a0a7c0:38497 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T01:55:02,452 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T01:55:02,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T01:55:02,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:02,459 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] zookeeper.ZKUtil(347): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T01:55:02,459 WARN [M:0;c6c0d7a0a7c0:38497 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T01:55:02,460 INFO [M:0;c6c0d7a0a7c0:38497 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/.lastflushedseqids 2024-11-17T01:55:02,468 INFO [M:0;c6c0d7a0a7c0:38497 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T01:55:02,469 INFO [M:0;c6c0d7a0a7c0:38497 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T01:55:02,470 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T01:55:02,470 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:55:02,470 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:55:02,470 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T01:55:02,470 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:55:02,470 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.93 KB heapSize=44.14 KB 2024-11-17T01:55:02,481 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fc6fe7ef36ce478c8f85ed7c7eb945de 2024-11-17T01:55:02,497 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc6fe7ef36ce478c8f85ed7c7eb945de is 82, key is hbase:meta,,1/info:regioninfo/1731808501750/Put/seqid=0 2024-11-17T01:55:02,498 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc6fe7ef36ce478c8f85ed7c7eb945de 2024-11-17T01:55:02,506 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 35fc9d7a390049b498bf34fb963692c3 2024-11-17T01:55:02,523 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35fc9d7a390049b498bf34fb963692c3 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731808499702/Put/seqid=0 2024-11-17T01:55:02,524 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.70 KB at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35fc9d7a390049b498bf34fb963692c3 2024-11-17T01:55:02,535 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e93f1b37ad7146f0b27e334b03e70a34 2024-11-17T01:55:02,543 INFO [RS:0;c6c0d7a0a7c0:39257 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T01:55:02,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:55:02,543 INFO [RS:0;c6c0d7a0a7c0:39257 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6c0d7a0a7c0,39257,1731808498125; zookeeper connection closed. 2024-11-17T01:55:02,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39257-0x10147ca23fd0001, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:55:02,543 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3fa1797 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3fa1797 2024-11-17T01:55:02,543 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-17T01:55:02,555 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e93f1b37ad7146f0b27e334b03e70a34 is 69, key is c6c0d7a0a7c0,37061,1731808501508/rs:state/1731808501592/Put/seqid=0 2024-11-17T01:55:02,556 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e93f1b37ad7146f0b27e334b03e70a34 2024-11-17T01:55:02,557 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e93f1b37ad7146f0b27e334b03e70a34 2024-11-17T01:55:02,558 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc6fe7ef36ce478c8f85ed7c7eb945de as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fc6fe7ef36ce478c8f85ed7c7eb945de 2024-11-17T01:55:02,560 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fc6fe7ef36ce478c8f85ed7c7eb945de, entries=14, sequenceid=90, filesize=5.9 K 2024-11-17T01:55:02,560 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35fc9d7a390049b498bf34fb963692c3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/35fc9d7a390049b498bf34fb963692c3 2024-11-17T01:55:02,562 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/35fc9d7a390049b498bf34fb963692c3, entries=11, sequenceid=90, filesize=7.8 K 2024-11-17T01:55:02,563 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e93f1b37ad7146f0b27e334b03e70a34 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e93f1b37ad7146f0b27e334b03e70a34 2024-11-17T01:55:02,564 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e93f1b37ad7146f0b27e334b03e70a34 2024-11-17T01:55:02,565 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/92f5de36-25db-64fc-5bb9-17731655cf87/43779786-0492-1e47-8285-247902c41eb8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e93f1b37ad7146f0b27e334b03e70a34, entries=3, sequenceid=90, filesize=5.3 K 2024-11-17T01:55:02,565 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.93 KB/35768, heapSize ~43.84 KB/44896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 95ms, sequenceid=90, compaction requested=false 2024-11-17T01:55:02,567 INFO [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:55:02,567 DEBUG [M:0;c6c0d7a0a7c0:38497 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731808502470Disabling compacts and flushes for region at 1731808502470Disabling writes for close at 1731808502470Obtaining lock to block concurrent updates at 1731808502470Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731808502470Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35768, getHeapSize=45136, getOffHeapSize=0, getCellsCount=111 at 1731808502471 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731808502471Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731808502471Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731808502497 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731808502497Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731808502500 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731808502522 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731808502523 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731808502527 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731808502554 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731808502554Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33149ad8: reopening flushed file at 1731808502558 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c30d151: reopening flushed file at 1731808502560 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13713982: reopening flushed file at 1731808502562 (+2 ms)Finished flush of dataSize ~34.93 KB/35768, heapSize ~43.84 KB/44896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 95ms, sequenceid=90, compaction requested=false at 1731808502565 (+3 ms)Writing region close event to WAL at 1731808502567 (+2 ms)Closed at 1731808502567 2024-11-17T01:55:02,569 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T01:55:02,569 INFO [M:0;c6c0d7a0a7c0:38497 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T01:55:02,569 INFO [M:0;c6c0d7a0a7c0:38497 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38497 2024-11-17T01:55:02,569 INFO [M:0;c6c0d7a0a7c0:38497 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T01:55:02,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:55:02,684 INFO [M:0;c6c0d7a0a7c0:38497 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T01:55:02,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38497-0x10147ca23fd0000, quorum=127.0.0.1:49311, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:55:02,688 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T01:55:02,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-17T01:55:02,689 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:55:02,695 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=69 (was 49) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=302 (was 311), ProcessCount=11 (was 11), AvailableMemoryMB=6938 (was 6126) - AvailableMemoryMB LEAK? -