2024-11-18 23:54:13,897 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-18 23:54:13,914 main DEBUG Took 0.015725 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 23:54:13,915 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 23:54:13,915 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 23:54:13,916 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 23:54:13,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,927 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 23:54:13,943 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,945 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,946 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,946 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,948 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,948 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,949 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,949 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,950 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,951 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,951 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,952 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,952 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,953 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,953 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,954 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,954 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,955 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,956 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,956 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:54:13,957 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,958 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 23:54:13,959 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:54:13,961 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 23:54:13,963 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 23:54:13,963 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 23:54:13,964 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 23:54:13,964 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 23:54:13,973 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 23:54:13,975 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 23:54:13,977 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 23:54:13,977 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 23:54:13,978 main DEBUG createAppenders(={Console}) 2024-11-18 23:54:13,978 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-18 23:54:13,978 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-18 23:54:13,979 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-18 23:54:13,979 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 23:54:13,979 main DEBUG OutputStream closed 2024-11-18 23:54:13,980 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 23:54:13,980 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 23:54:13,980 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-18 23:54:14,046 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 23:54:14,049 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 23:54:14,049 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 23:54:14,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 23:54:14,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 23:54:14,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 23:54:14,052 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 23:54:14,052 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 23:54:14,052 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 23:54:14,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 23:54:14,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 23:54:14,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 23:54:14,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 23:54:14,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 23:54:14,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 23:54:14,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 23:54:14,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 23:54:14,055 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 23:54:14,058 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 23:54:14,058 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-18 23:54:14,058 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 23:54:14,059 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-18T23:54:14,073 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-18 23:54:14,076 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 23:54:14,076 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T23:54:14,337 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e 2024-11-18T23:54:14,364 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=99, ProcessCount=11, AvailableMemoryMB=5234 2024-11-18T23:54:14,477 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T23:54:14,577 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:14,592 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/cluster_3c27961c-1a81-c47f-de43-4979733105c7, deleteOnExit=true 2024-11-18T23:54:14,681 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/cluster_3c27961c-1a81-c47f-de43-4979733105c7/zookeeper_0, clientPort=59958, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/cluster_3c27961c-1a81-c47f-de43-4979733105c7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/cluster_3c27961c-1a81-c47f-de43-4979733105c7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T23:54:14,696 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59958 2024-11-18T23:54:14,696 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:14,782 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d with version=8 2024-11-18T23:54:14,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/hbase-staging 2024-11-18T23:54:14,882 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T23:54:15,155 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9c732c6543ff:0 server-side Connection retries=45 2024-11-18T23:54:15,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:15,167 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:15,176 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:54:15,177 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:15,177 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:54:15,317 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T23:54:15,408 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:54:15,456 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T23:54:15,467 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T23:54:15,472 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:54:15,513 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21339 (auto-detected) 2024-11-18T23:54:15,514 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T23:54:15,534 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41843 2024-11-18T23:54:15,557 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41843 connecting to ZooKeeper ensemble=127.0.0.1:59958 2024-11-18T23:54:15,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:418430x0, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:54:15,594 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41843-0x1010845b7b00000 connected 2024-11-18T23:54:15,622 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:15,623 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:15,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:15,635 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d, hbase.cluster.distributed=false 2024-11-18T23:54:15,659 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:54:15,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41843 2024-11-18T23:54:15,665 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41843 2024-11-18T23:54:15,665 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41843 2024-11-18T23:54:15,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41843 2024-11-18T23:54:15,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41843 2024-11-18T23:54:15,754 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45 2024-11-18T23:54:15,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:15,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:15,757 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:54:15,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:15,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:54:15,760 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T23:54:15,762 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:54:15,764 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46031 2024-11-18T23:54:15,766 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46031 connecting to ZooKeeper ensemble=127.0.0.1:59958 2024-11-18T23:54:15,766 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:15,767 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:15,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460310x0, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:54:15,772 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:460310x0, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:15,772 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46031-0x1010845b7b00001 connected 2024-11-18T23:54:15,778 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T23:54:15,791 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/bucket.cache 2024-11-18T23:54:15,796 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T23:54:15,797 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T23:54:15,799 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T23:54:15,800 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T23:54:15,800 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T23:54:15,808 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T23:54:15,811 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T23:54:15,812 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/bucket.persistence is complete. 2024-11-18T23:54:15,814 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T23:54:15,817 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T23:54:15,822 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:54:15,822 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46031 2024-11-18T23:54:15,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46031 2024-11-18T23:54:15,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46031 2024-11-18T23:54:15,829 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46031 2024-11-18T23:54:15,829 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46031 2024-11-18T23:54:15,849 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45 2024-11-18T23:54:15,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:15,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:15,850 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:54:15,850 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:15,850 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:54:15,850 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T23:54:15,850 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:54:15,851 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33549 2024-11-18T23:54:15,853 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33549 connecting to ZooKeeper ensemble=127.0.0.1:59958 2024-11-18T23:54:15,854 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:15,855 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:15,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335490x0, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:54:15,860 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:335490x0, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:15,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33549-0x1010845b7b00002 connected 2024-11-18T23:54:15,861 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T23:54:15,861 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/bucket.cache 2024-11-18T23:54:15,862 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T23:54:15,862 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T23:54:15,863 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T23:54:15,863 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T23:54:15,863 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T23:54:15,864 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T23:54:15,864 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/bucket.persistence is complete. 2024-11-18T23:54:15,865 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T23:54:15,866 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T23:54:15,867 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T23:54:15,869 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:54:15,871 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33549 2024-11-18T23:54:15,873 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33549 2024-11-18T23:54:15,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33549 2024-11-18T23:54:15,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33549 2024-11-18T23:54:15,877 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33549 2024-11-18T23:54:15,899 DEBUG [M:0;9c732c6543ff:41843 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9c732c6543ff:41843 2024-11-18T23:54:15,900 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9c732c6543ff,41843,1731974054954 2024-11-18T23:54:15,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:15,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:15,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:15,913 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9c732c6543ff,41843,1731974054954 2024-11-18T23:54:15,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T23:54:15,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:15,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T23:54:15,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:15,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:15,940 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T23:54:15,942 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9c732c6543ff,41843,1731974054954 from backup master directory 2024-11-18T23:54:15,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9c732c6543ff,41843,1731974054954 2024-11-18T23:54:15,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:15,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:15,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:15,946 WARN [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:54:15,947 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9c732c6543ff,41843,1731974054954 2024-11-18T23:54:15,950 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T23:54:15,952 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T23:54:15,963 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/hbase.id] with ID: f0420cc3-e1db-48ec-aaa3-f374dbea2fdf 2024-11-18T23:54:15,963 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/.tmp/hbase.id 2024-11-18T23:54:15,970 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/hbase.id] 2024-11-18T23:54:16,012 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:16,018 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T23:54:16,047 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-18T23:54:16,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,082 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:54:16,085 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T23:54:16,132 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:54:16,139 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:16,198 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store 2024-11-18T23:54:16,222 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T23:54:16,227 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:16,229 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T23:54:16,229 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:16,230 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:16,232 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T23:54:16,232 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:16,233 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:16,233 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731974056229Disabling compacts and flushes for region at 1731974056229Disabling writes for close at 1731974056232 (+3 ms)Writing region close event to WAL at 1731974056232Closed at 1731974056232 2024-11-18T23:54:16,234 WARN [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/.initializing 2024-11-18T23:54:16,234 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/WALs/9c732c6543ff,41843,1731974054954 2024-11-18T23:54:16,251 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:54:16,269 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C41843%2C1731974054954, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/WALs/9c732c6543ff,41843,1731974054954, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/oldWALs, maxLogs=10 2024-11-18T23:54:16,308 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/WALs/9c732c6543ff,41843,1731974054954/9c732c6543ff%2C41843%2C1731974054954.1731974056274 2024-11-18T23:54:16,309 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:16,309 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:16,310 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:16,314 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,315 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,354 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,386 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T23:54:16,391 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:16,393 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:16,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,400 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T23:54:16,400 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:16,400 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:16,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,406 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T23:54:16,406 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:16,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:16,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,413 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T23:54:16,413 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:16,413 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:16,414 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,417 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,418 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,421 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,421 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,424 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:54:16,426 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:16,434 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:54:16,435 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75416071, jitterRate=0.1237870305776596}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:54:16,441 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731974056328Initializing all the Stores at 1731974056328Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974056329 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974056329Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974056330 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974056330Cleaning up temporary data from old regions at 1731974056422 (+92 ms)Region opened successfully at 1731974056441 (+19 ms) 2024-11-18T23:54:16,443 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T23:54:16,476 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e5a966d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0 2024-11-18T23:54:16,512 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T23:54:16,524 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T23:54:16,525 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T23:54:16,528 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T23:54:16,529 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T23:54:16,535 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-18T23:54:16,535 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T23:54:16,561 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T23:54:16,571 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T23:54:16,573 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T23:54:16,576 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T23:54:16,578 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T23:54:16,580 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T23:54:16,583 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T23:54:16,589 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T23:54:16,591 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T23:54:16,593 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T23:54:16,594 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T23:54:16,619 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T23:54:16,621 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T23:54:16,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:16,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:16,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:16,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,643 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9c732c6543ff,41843,1731974054954, sessionid=0x1010845b7b00000, setting cluster-up flag (Was=false) 2024-11-18T23:54:16,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,664 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T23:54:16,666 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,41843,1731974054954 2024-11-18T23:54:16,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:16,677 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T23:54:16,679 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,41843,1731974054954 2024-11-18T23:54:16,684 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T23:54:16,757 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:16,768 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T23:54:16,775 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T23:54:16,783 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer(746): ClusterId : f0420cc3-e1db-48ec-aaa3-f374dbea2fdf 2024-11-18T23:54:16,783 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(746): ClusterId : f0420cc3-e1db-48ec-aaa3-f374dbea2fdf 2024-11-18T23:54:16,781 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9c732c6543ff,41843,1731974054954 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T23:54:16,785 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T23:54:16,785 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T23:54:16,791 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:16,791 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T23:54:16,791 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:16,791 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T23:54:16,791 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T23:54:16,791 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T23:54:16,791 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:16,792 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:16,792 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9c732c6543ff:0, corePoolSize=10, maxPoolSize=10 2024-11-18T23:54:16,792 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:16,792 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:54:16,792 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:16,795 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T23:54:16,795 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T23:54:16,795 DEBUG [RS:0;9c732c6543ff:46031 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a684d24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0 2024-11-18T23:54:16,796 DEBUG [RS:1;9c732c6543ff:33549 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a9e9f48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0 2024-11-18T23:54:16,797 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731974086797 2024-11-18T23:54:16,797 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:16,797 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T23:54:16,798 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T23:54:16,800 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T23:54:16,803 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:16,804 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T23:54:16,804 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T23:54:16,804 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T23:54:16,805 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T23:54:16,805 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T23:54:16,806 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:16,815 DEBUG [RS:0;9c732c6543ff:46031 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9c732c6543ff:46031 2024-11-18T23:54:16,816 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T23:54:16,818 DEBUG [RS:1;9c732c6543ff:33549 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9c732c6543ff:33549 2024-11-18T23:54:16,818 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T23:54:16,818 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T23:54:16,818 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T23:54:16,818 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T23:54:16,819 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T23:54:16,819 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T23:54:16,819 DEBUG [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T23:54:16,819 DEBUG [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T23:54:16,821 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T23:54:16,822 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T23:54:16,823 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,41843,1731974054954 with port=33549, startcode=1731974055848 2024-11-18T23:54:16,823 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,41843,1731974054954 with port=46031, startcode=1731974055714 2024-11-18T23:54:16,825 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974056824,5,FailOnTimeoutGroup] 2024-11-18T23:54:16,826 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974056825,5,FailOnTimeoutGroup] 2024-11-18T23:54:16,826 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:16,826 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T23:54:16,828 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:16,828 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:16,838 DEBUG [RS:0;9c732c6543ff:46031 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T23:54:16,838 DEBUG [RS:1;9c732c6543ff:33549 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T23:54:16,840 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T23:54:16,841 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d 2024-11-18T23:54:16,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:16,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T23:54:16,857 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T23:54:16,857 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:16,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:16,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T23:54:16,862 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T23:54:16,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:16,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:16,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T23:54:16,867 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T23:54:16,867 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:16,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:16,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T23:54:16,871 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T23:54:16,871 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:16,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:16,872 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T23:54:16,873 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740 2024-11-18T23:54:16,873 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740 2024-11-18T23:54:16,874 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T23:54:16,874 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T23:54:16,874 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:54:16,875 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T23:54:16,888 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:54:16,888 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65127439, jitterRate=-0.029525533318519592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:54:16,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731974056851Initializing all the Stores at 1731974056852 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974056852Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974056852Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974056852Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974056852Cleaning up temporary data from old regions at 1731974056874 (+22 ms)Region opened successfully at 1731974056893 (+19 ms) 2024-11-18T23:54:16,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T23:54:16,894 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T23:54:16,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T23:54:16,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T23:54:16,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T23:54:16,896 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T23:54:16,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731974056893Disabling compacts and flushes for region at 1731974056893Disabling writes for close at 1731974056894 (+1 ms)Writing region close event to WAL at 1731974056896 (+2 ms)Closed at 1731974056896 2024-11-18T23:54:16,899 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:16,899 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T23:54:16,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T23:54:16,915 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46381, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T23:54:16,915 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51227, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T23:54:16,915 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:54:16,917 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T23:54:16,922 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,46031,1731974055714 2024-11-18T23:54:16,925 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41843 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,46031,1731974055714 2024-11-18T23:54:16,934 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,33549,1731974055848 2024-11-18T23:54:16,934 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41843 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,33549,1731974055848 2024-11-18T23:54:16,939 DEBUG [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d 2024-11-18T23:54:16,939 DEBUG [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d 2024-11-18T23:54:16,939 DEBUG [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T23:54:16,939 DEBUG [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T23:54:16,939 DEBUG [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T23:54:16,939 DEBUG [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T23:54:16,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:54:16,948 DEBUG [RS:0;9c732c6543ff:46031 {}] zookeeper.ZKUtil(111): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,46031,1731974055714 2024-11-18T23:54:16,948 DEBUG [RS:1;9c732c6543ff:33549 {}] zookeeper.ZKUtil(111): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,33549,1731974055848 2024-11-18T23:54:16,948 WARN [RS:0;9c732c6543ff:46031 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:54:16,948 WARN [RS:1;9c732c6543ff:33549 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:54:16,948 INFO [RS:0;9c732c6543ff:46031 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:16,948 INFO [RS:1;9c732c6543ff:33549 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:16,948 DEBUG [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,46031,1731974055714 2024-11-18T23:54:16,949 DEBUG [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,33549,1731974055848 2024-11-18T23:54:16,950 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,46031,1731974055714] 2024-11-18T23:54:16,950 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,33549,1731974055848] 2024-11-18T23:54:16,974 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T23:54:16,974 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T23:54:16,988 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T23:54:16,988 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T23:54:16,993 INFO [RS:1;9c732c6543ff:33549 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:54:16,993 INFO [RS:0;9c732c6543ff:46031 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:54:16,993 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:16,993 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:16,994 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T23:54:16,994 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T23:54:17,000 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T23:54:17,000 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T23:54:17,002 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,002 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,002 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,002 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,002 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,002 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,002 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,002 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,003 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,003 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,003 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,003 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,003 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:54:17,003 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:54:17,003 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,003 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,004 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,004 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,004 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,004 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,004 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,004 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,004 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,004 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,004 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,004 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:17,005 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:17,005 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:17,005 DEBUG [RS:1;9c732c6543ff:33549 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:17,005 DEBUG [RS:0;9c732c6543ff:46031 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:17,006 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,006 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,007 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,007 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,007 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,007 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,33549,1731974055848-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:54:17,007 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,007 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,008 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,008 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,008 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,008 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,46031,1731974055714-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:54:17,028 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T23:54:17,030 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,33549,1731974055848-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,030 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,030 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.Replication(171): 9c732c6543ff,33549,1731974055848 started 2024-11-18T23:54:17,031 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T23:54:17,031 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,46031,1731974055714-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,031 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,032 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.Replication(171): 9c732c6543ff,46031,1731974055714 started 2024-11-18T23:54:17,050 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,050 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,33549,1731974055848, RpcServer on 9c732c6543ff/172.17.0.2:33549, sessionid=0x1010845b7b00002 2024-11-18T23:54:17,051 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T23:54:17,052 DEBUG [RS:1;9c732c6543ff:33549 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,33549,1731974055848 2024-11-18T23:54:17,052 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,33549,1731974055848' 2024-11-18T23:54:17,052 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T23:54:17,053 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T23:54:17,054 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T23:54:17,054 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T23:54:17,054 DEBUG [RS:1;9c732c6543ff:33549 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,33549,1731974055848 2024-11-18T23:54:17,054 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,33549,1731974055848' 2024-11-18T23:54:17,055 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T23:54:17,055 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T23:54:17,056 DEBUG [RS:1;9c732c6543ff:33549 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T23:54:17,056 INFO [RS:1;9c732c6543ff:33549 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T23:54:17,056 INFO [RS:1;9c732c6543ff:33549 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T23:54:17,056 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,057 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,46031,1731974055714, RpcServer on 9c732c6543ff/172.17.0.2:46031, sessionid=0x1010845b7b00001 2024-11-18T23:54:17,057 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T23:54:17,057 DEBUG [RS:0;9c732c6543ff:46031 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,46031,1731974055714 2024-11-18T23:54:17,057 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,46031,1731974055714' 2024-11-18T23:54:17,057 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T23:54:17,058 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T23:54:17,059 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T23:54:17,059 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T23:54:17,059 DEBUG [RS:0;9c732c6543ff:46031 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,46031,1731974055714 2024-11-18T23:54:17,059 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,46031,1731974055714' 2024-11-18T23:54:17,059 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T23:54:17,060 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T23:54:17,061 DEBUG [RS:0;9c732c6543ff:46031 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T23:54:17,061 INFO [RS:0;9c732c6543ff:46031 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T23:54:17,061 INFO [RS:0;9c732c6543ff:46031 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T23:54:17,068 WARN [9c732c6543ff:41843 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T23:54:17,162 INFO [RS:1;9c732c6543ff:33549 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:54:17,162 INFO [RS:0;9c732c6543ff:46031 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:54:17,163 INFO [RS:1;9c732c6543ff:33549 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9c732c6543ff%2C33549%2C1731974055848, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,33549,1731974055848, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs, maxLogs=55 2024-11-18T23:54:17,163 INFO [RS:0;9c732c6543ff:46031 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9c732c6543ff%2C46031%2C1731974055714, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,46031,1731974055714, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs, maxLogs=55 2024-11-18T23:54:17,186 INFO [RS:1;9c732c6543ff:33549 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,33549,1731974055848/9c732c6543ff%2C33549%2C1731974055848.1731974057166 2024-11-18T23:54:17,187 DEBUG [RS:1;9c732c6543ff:33549 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:17,188 INFO [RS:0;9c732c6543ff:46031 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,46031,1731974055714/9c732c6543ff%2C46031%2C1731974055714.1731974057167 2024-11-18T23:54:17,189 DEBUG [RS:0;9c732c6543ff:46031 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:17,321 DEBUG [9c732c6543ff:41843 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-18T23:54:17,332 DEBUG [9c732c6543ff:41843 {}] balancer.BalancerClusterState(204): Hosts are {9c732c6543ff=0} racks are {/default-rack=0} 2024-11-18T23:54:17,336 DEBUG [9c732c6543ff:41843 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T23:54:17,336 DEBUG [9c732c6543ff:41843 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T23:54:17,336 DEBUG [9c732c6543ff:41843 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T23:54:17,336 DEBUG [9c732c6543ff:41843 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T23:54:17,336 INFO [9c732c6543ff:41843 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T23:54:17,337 INFO [9c732c6543ff:41843 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T23:54:17,337 DEBUG [9c732c6543ff:41843 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T23:54:17,344 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9c732c6543ff,46031,1731974055714 2024-11-18T23:54:17,350 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,46031,1731974055714, state=OPENING 2024-11-18T23:54:17,357 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T23:54:17,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:17,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:17,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:17,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:17,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:17,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:17,362 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:54:17,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,46031,1731974055714}] 2024-11-18T23:54:17,537 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:54:17,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51481, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:54:17,548 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T23:54:17,549 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:17,550 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T23:54:17,551 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9c732c6543ff%2C46031%2C1731974055714.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,46031,1731974055714, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs, maxLogs=55 2024-11-18T23:54:17,571 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,46031,1731974055714/9c732c6543ff%2C46031%2C1731974055714.meta.1731974057552.meta 2024-11-18T23:54:17,572 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:17,572 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:17,574 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T23:54:17,577 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T23:54:17,581 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T23:54:17,585 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T23:54:17,586 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:17,586 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T23:54:17,586 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T23:54:17,588 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T23:54:17,589 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T23:54:17,589 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:17,589 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:17,589 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T23:54:17,590 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T23:54:17,590 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:17,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:17,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T23:54:17,591 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T23:54:17,592 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:17,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:17,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T23:54:17,593 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T23:54:17,593 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:17,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:17,593 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T23:54:17,594 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740 2024-11-18T23:54:17,595 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740 2024-11-18T23:54:17,595 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T23:54:17,595 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T23:54:17,596 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:54:17,597 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T23:54:17,597 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64579313, jitterRate=-0.03769324719905853}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:54:17,598 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T23:54:17,599 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731974057587Writing region info on filesystem at 1731974057587Initializing all the Stores at 1731974057587Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974057587Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974057588 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974057588Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974057588Cleaning up temporary data from old regions at 1731974057595 (+7 ms)Running coprocessor post-open hooks at 1731974057598 (+3 ms)Region opened successfully at 1731974057599 (+1 ms) 2024-11-18T23:54:17,605 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731974057528 2024-11-18T23:54:17,617 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,46031,1731974055714 2024-11-18T23:54:17,619 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T23:54:17,619 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,46031,1731974055714, state=OPEN 2024-11-18T23:54:17,619 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T23:54:17,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:17,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:17,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:17,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:17,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:17,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:17,622 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9c732c6543ff,46031,1731974055714 2024-11-18T23:54:17,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T23:54:17,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,46031,1731974055714 in 259 msec 2024-11-18T23:54:17,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T23:54:17,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 721 msec 2024-11-18T23:54:17,630 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:17,631 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T23:54:17,653 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:54:17,654 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,46031,1731974055714, seqNum=-1] 2024-11-18T23:54:17,675 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:54:17,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44233, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:54:17,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 983 msec 2024-11-18T23:54:17,696 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731974057695, completionTime=-1 2024-11-18T23:54:17,698 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-18T23:54:17,698 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T23:54:17,724 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T23:54:17,725 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731974117724 2024-11-18T23:54:17,725 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731974177725 2024-11-18T23:54:17,725 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-18T23:54:17,728 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,41843,1731974054954-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,729 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,41843,1731974054954-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,729 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,41843,1731974054954-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,731 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9c732c6543ff:41843, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,731 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,732 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,737 DEBUG [master/9c732c6543ff:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T23:54:17,759 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.812sec 2024-11-18T23:54:17,761 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T23:54:17,762 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T23:54:17,763 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T23:54:17,764 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T23:54:17,764 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T23:54:17,765 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,41843,1731974054954-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:54:17,765 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,41843,1731974054954-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T23:54:17,768 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T23:54:17,769 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T23:54:17,770 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,41843,1731974054954-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:17,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd87de0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:54:17,811 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T23:54:17,811 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T23:54:17,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9c732c6543ff,41843,-1 for getting cluster id 2024-11-18T23:54:17,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:54:17,829 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f0420cc3-e1db-48ec-aaa3-f374dbea2fdf' 2024-11-18T23:54:17,831 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:54:17,831 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f0420cc3-e1db-48ec-aaa3-f374dbea2fdf" 2024-11-18T23:54:17,832 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5166ce67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:54:17,832 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9c732c6543ff,41843,-1] 2024-11-18T23:54:17,835 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:54:17,837 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:17,838 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:54:17,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769ae1ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:54:17,841 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:54:17,848 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,46031,1731974055714, seqNum=-1] 2024-11-18T23:54:17,848 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:54:17,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:54:17,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9c732c6543ff,41843,1731974054954 2024-11-18T23:54:17,873 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T23:54:17,878 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9c732c6543ff,41843,1731974054954 2024-11-18T23:54:17,880 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d7ed99 2024-11-18T23:54:17,881 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T23:54:17,884 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T23:54:17,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41843 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731974057870', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:54:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41843 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731974057870 2024-11-18T23:54:17,897 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731974057870 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T23:54:17,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41843 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731974057870" procId is: 4 2024-11-18T23:54:17,899 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:17,901 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731974057870 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T23:54:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:17,915 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731974057870-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 557941f9166b64eb2ec3d166e365717f, NAME => 'testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731974057870', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d 2024-11-18T23:54:17,921 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731974057870-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:17,922 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731974057870-pool-0 {}] regionserver.HRegion(1722): Closing 557941f9166b64eb2ec3d166e365717f, disabling compactions & flushes 2024-11-18T23:54:17,922 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731974057870-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:17,922 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731974057870-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:17,922 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731974057870-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. after waiting 0 ms 2024-11-18T23:54:17,922 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731974057870-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:17,922 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731974057870-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:17,922 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731974057870-pool-0 {}] regionserver.HRegion(1676): Region close journal for 557941f9166b64eb2ec3d166e365717f: Waiting for close lock at 1731974057922Disabling compacts and flushes for region at 1731974057922Disabling writes for close at 1731974057922Writing region close event to WAL at 1731974057922Closed at 1731974057922 2024-11-18T23:54:17,923 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731974057870 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T23:54:17,929 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731974057924"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731974057924"}]},"ts":"1731974057924"} 2024-11-18T23:54:17,934 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T23:54:17,935 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731974057870 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T23:54:17,937 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731974057870","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974057935"}]},"ts":"1731974057935"} 2024-11-18T23:54:17,941 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731974057870, state=ENABLING in hbase:meta 2024-11-18T23:54:17,942 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9c732c6543ff=0} racks are {/default-rack=0} 2024-11-18T23:54:17,942 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T23:54:17,943 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T23:54:17,943 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T23:54:17,943 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T23:54:17,943 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T23:54:17,943 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T23:54:17,943 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T23:54:17,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731974057870, region=557941f9166b64eb2ec3d166e365717f, ASSIGN}] 2024-11-18T23:54:17,945 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731974057870, region=557941f9166b64eb2ec3d166e365717f, ASSIGN 2024-11-18T23:54:17,946 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731974057870, region=557941f9166b64eb2ec3d166e365717f, ASSIGN; state=OFFLINE, location=9c732c6543ff,33549,1731974055848; forceNewPlan=false, retain=false 2024-11-18T23:54:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:18,099 INFO [9c732c6543ff:41843 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T23:54:18,100 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=557941f9166b64eb2ec3d166e365717f, regionState=OPENING, regionLocation=9c732c6543ff,33549,1731974055848 2024-11-18T23:54:18,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731974057870, region=557941f9166b64eb2ec3d166e365717f, ASSIGN because future has completed 2024-11-18T23:54:18,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 557941f9166b64eb2ec3d166e365717f, server=9c732c6543ff,33549,1731974055848}] 2024-11-18T23:54:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:18,257 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:54:18,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37467, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:54:18,263 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:18,263 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 557941f9166b64eb2ec3d166e365717f, NAME => 'testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:18,264 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731974057870 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:18,264 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:18,264 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:18,264 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:18,265 INFO [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:18,269 INFO [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 557941f9166b64eb2ec3d166e365717f columnFamilyName family 2024-11-18T23:54:18,269 DEBUG [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:18,270 INFO [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] regionserver.HStore(327): Store=557941f9166b64eb2ec3d166e365717f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:18,270 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:18,271 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:18,271 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:18,271 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:18,271 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:18,272 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:18,278 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:54:18,278 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 557941f9166b64eb2ec3d166e365717f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65922720, jitterRate=-0.017674922943115234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:54:18,278 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:18,279 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 557941f9166b64eb2ec3d166e365717f: Running coprocessor pre-open hook at 1731974058264Writing region info on filesystem at 1731974058264Initializing all the Stores at 1731974058264Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974058264Cleaning up temporary data from old regions at 1731974058271 (+7 ms)Running coprocessor post-open hooks at 1731974058279 (+8 ms)Region opened successfully at 1731974058279 2024-11-18T23:54:18,280 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f., pid=6, masterSystemTime=1731974058257 2024-11-18T23:54:18,284 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:18,284 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:18,284 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=557941f9166b64eb2ec3d166e365717f, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,33549,1731974055848 2024-11-18T23:54:18,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 557941f9166b64eb2ec3d166e365717f, server=9c732c6543ff,33549,1731974055848 because future has completed 2024-11-18T23:54:18,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T23:54:18,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 557941f9166b64eb2ec3d166e365717f, server=9c732c6543ff,33549,1731974055848 in 185 msec 2024-11-18T23:54:18,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T23:54:18,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731974057870, region=557941f9166b64eb2ec3d166e365717f, ASSIGN in 347 msec 2024-11-18T23:54:18,294 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731974057870 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T23:54:18,294 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731974057870","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974058294"}]},"ts":"1731974058294"} 2024-11-18T23:54:18,297 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731974057870, state=ENABLED in hbase:meta 2024-11-18T23:54:18,298 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731974057870 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T23:54:18,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731974057870 in 406 msec 2024-11-18T23:54:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:18,536 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731974057870 completed 2024-11-18T23:54:18,536 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731974057870 get assigned. Timeout = 60000ms 2024-11-18T23:54:18,539 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T23:54:18,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731974057870 assigned to meta. Checking AM states. 2024-11-18T23:54:18,548 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T23:54:18,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731974057870 assigned. 2024-11-18T23:54:18,558 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731974057870', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f., hostname=9c732c6543ff,33549,1731974055848, seqNum=2] 2024-11-18T23:54:18,559 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:54:18,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:54:18,570 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 557941f9166b64eb2ec3d166e365717f 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-18T23:54:18,600 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fd16d73d659549f692fa0d337a88292f 2024-11-18T23:54:18,645 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/.tmp/family/fd16d73d659549f692fa0d337a88292f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-18T23:54:18,653 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/.tmp/family/fd16d73d659549f692fa0d337a88292f 2024-11-18T23:54:18,690 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/.tmp/family/fd16d73d659549f692fa0d337a88292f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f 2024-11-18T23:54:18,691 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f, delay=996 ms 2024-11-18T23:54:18,693 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f, entries=2, sequenceid=6, filesize=4.9 K 2024-11-18T23:54:18,699 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 557941f9166b64eb2ec3d166e365717f in 128ms, sequenceid=6, compaction requested=false 2024-11-18T23:54:18,700 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731974057870' 2024-11-18T23:54:18,702 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 557941f9166b64eb2ec3d166e365717f: 2024-11-18T23:54:19,690 DEBUG [hfile-prefetch-1731974058691 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fd16d73d659549f692fa0d337a88292f_0. 2024-11-18T23:54:19,694 DEBUG [hfile-prefetch-1731974058691 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fd16d73d659549f692fa0d337a88292f_117. 2024-11-18T23:54:19,694 DEBUG [hfile-prefetch-1731974058691 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f, with total blocks 2, and data blocks 1 2024-11-18T23:54:19,694 DEBUG [hfile-prefetch-1731974058691 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-18T23:54:19,695 DEBUG [hfile-prefetch-1731974058691 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-18T23:54:19,695 DEBUG [hfile-prefetch-1731974058691 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f. Will sleep 100ms and try the verification again. 2024-11-18T23:54:19,703 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T23:54:19,703 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T23:54:19,704 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T23:54:19,704 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T23:54:19,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41843 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=557941f9166b64eb2ec3d166e365717f, source=9c732c6543ff,33549,1731974055848, destination=9c732c6543ff,46031,1731974055714, warming up region on 9c732c6543ff,46031,1731974055714 2024-11-18T23:54:19,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41843 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=557941f9166b64eb2ec3d166e365717f, source=9c732c6543ff,33549,1731974055848, destination=9c732c6543ff,46031,1731974055714, running balancer 2024-11-18T23:54:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41843 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731974057870, region=557941f9166b64eb2ec3d166e365717f, REOPEN/MOVE 2024-11-18T23:54:19,716 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731974057870, region=557941f9166b64eb2ec3d166e365717f, REOPEN/MOVE 2024-11-18T23:54:19,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] regionserver.HRegion(7855): Warmup {ENCODED => 557941f9166b64eb2ec3d166e365717f, NAME => 'testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:19,718 INFO [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:19,718 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=557941f9166b64eb2ec3d166e365717f, regionState=CLOSING, regionLocation=9c732c6543ff,33549,1731974055848 2024-11-18T23:54:19,719 INFO [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 557941f9166b64eb2ec3d166e365717f columnFamilyName family 2024-11-18T23:54:19,719 DEBUG [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:19,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731974057870, region=557941f9166b64eb2ec3d166e365717f, REOPEN/MOVE because future has completed 2024-11-18T23:54:19,722 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-18T23:54:19,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 557941f9166b64eb2ec3d166e365717f, server=9c732c6543ff,33549,1731974055848}] 2024-11-18T23:54:19,723 DEBUG [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f 2024-11-18T23:54:19,723 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T23:54:19,723 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731974059703 2024-11-18T23:54:19,723 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T23:54:19,723 INFO [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] regionserver.HStore(327): Store=557941f9166b64eb2ec3d166e365717f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] regionserver.HRegion(1722): Closing 557941f9166b64eb2ec3d166e365717f, disabling compactions & flushes 2024-11-18T23:54:19,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. after waiting 0 ms 2024-11-18T23:54:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:19,725 DEBUG [StoreFileCloser-557941f9166b64eb2ec3d166e365717f-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f 2024-11-18T23:54:19,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] regionserver.HRegion(1676): Region close journal for 557941f9166b64eb2ec3d166e365717f: Waiting for close lock at 1731974059723Disabling compacts and flushes for region at 1731974059723Disabling writes for close at 1731974059724 (+1 ms)Writing region close event to WAL at 1731974059725 (+1 ms)Closed at 1731974059725 2024-11-18T23:54:19,727 WARN [hfile-prefetch-1731974058691 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-18T23:54:19,728 DEBUG [hfile-prefetch-1731974058691 {}] hfile.PrefetchExecutor(121): Prefetch completed for fd16d73d659549f692fa0d337a88292f 2024-11-18T23:54:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41843 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-18T23:54:19,881 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:19,881 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-18T23:54:19,882 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 557941f9166b64eb2ec3d166e365717f, disabling compactions & flushes 2024-11-18T23:54:19,882 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:19,882 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:19,882 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. after waiting 0 ms 2024-11-18T23:54:19,882 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:19,884 DEBUG [StoreFileCloser-557941f9166b64eb2ec3d166e365717f-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file fd16d73d659549f692fa0d337a88292f, starting offset: 0, end offset: 9223372036854775807 2024-11-18T23:54:19,885 DEBUG [StoreFileCloser-557941f9166b64eb2ec3d166e365717f-family-1 {}] bucket.BucketCache(856): removed key fd16d73d659549f692fa0d337a88292f_0 from back map with offset lock 2094080 in the evict process 2024-11-18T23:54:19,890 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-18T23:54:19,892 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:19,892 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 557941f9166b64eb2ec3d166e365717f: Waiting for close lock at 1731974059882Running coprocessor pre-close hooks at 1731974059882Disabling compacts and flushes for region at 1731974059882Disabling writes for close at 1731974059882Writing region close event to WAL at 1731974059885 (+3 ms)Running coprocessor post-close hooks at 1731974059891 (+6 ms)Closed at 1731974059892 (+1 ms) 2024-11-18T23:54:19,892 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 557941f9166b64eb2ec3d166e365717f move to 9c732c6543ff,46031,1731974055714 record at close sequenceid=6 2024-11-18T23:54:19,896 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:19,896 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=557941f9166b64eb2ec3d166e365717f, regionState=CLOSED 2024-11-18T23:54:19,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 557941f9166b64eb2ec3d166e365717f, server=9c732c6543ff,33549,1731974055848 because future has completed 2024-11-18T23:54:19,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T23:54:19,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 557941f9166b64eb2ec3d166e365717f, server=9c732c6543ff,33549,1731974055848 in 178 msec 2024-11-18T23:54:19,902 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731974057870, region=557941f9166b64eb2ec3d166e365717f, REOPEN/MOVE; state=CLOSED, location=9c732c6543ff,46031,1731974055714; forceNewPlan=false, retain=false 2024-11-18T23:54:19,923 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T23:54:19,923 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T23:54:19,923 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T23:54:19,923 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T23:54:19,927 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T23:54:19,927 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731974059923 2024-11-18T23:54:19,927 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T23:54:20,053 INFO [9c732c6543ff:41843 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T23:54:20,053 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=557941f9166b64eb2ec3d166e365717f, regionState=OPENING, regionLocation=9c732c6543ff,46031,1731974055714 2024-11-18T23:54:20,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731974057870, region=557941f9166b64eb2ec3d166e365717f, REOPEN/MOVE because future has completed 2024-11-18T23:54:20,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 557941f9166b64eb2ec3d166e365717f, server=9c732c6543ff,46031,1731974055714}] 2024-11-18T23:54:20,212 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:20,212 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 557941f9166b64eb2ec3d166e365717f, NAME => 'testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:20,213 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731974057870 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,213 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:20,213 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,213 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,214 INFO [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,215 INFO [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 557941f9166b64eb2ec3d166e365717f columnFamilyName family 2024-11-18T23:54:20,215 DEBUG [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:20,217 DEBUG [StoreFileOpener-557941f9166b64eb2ec3d166e365717f-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f, delay=959 ms 2024-11-18T23:54:20,218 DEBUG [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f 2024-11-18T23:54:20,219 INFO [StoreOpener-557941f9166b64eb2ec3d166e365717f-1 {}] regionserver.HStore(327): Store=557941f9166b64eb2ec3d166e365717f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:20,219 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,220 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,220 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,220 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,220 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,225 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,225 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 557941f9166b64eb2ec3d166e365717f; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67105341, jitterRate=-5.249679088592529E-5}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:54:20,225 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,226 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 557941f9166b64eb2ec3d166e365717f: Running coprocessor pre-open hook at 1731974060213Writing region info on filesystem at 1731974060213Initializing all the Stores at 1731974060214 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974060214Cleaning up temporary data from old regions at 1731974060220 (+6 ms)Running coprocessor post-open hooks at 1731974060225 (+5 ms)Region opened successfully at 1731974060226 (+1 ms) 2024-11-18T23:54:20,227 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f., pid=9, masterSystemTime=1731974060210 2024-11-18T23:54:20,229 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:20,229 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:20,230 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=557941f9166b64eb2ec3d166e365717f, regionState=OPEN, openSeqNum=10, regionLocation=9c732c6543ff,46031,1731974055714 2024-11-18T23:54:20,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 557941f9166b64eb2ec3d166e365717f, server=9c732c6543ff,46031,1731974055714 because future has completed 2024-11-18T23:54:20,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-18T23:54:20,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 557941f9166b64eb2ec3d166e365717f, server=9c732c6543ff,46031,1731974055714 in 177 msec 2024-11-18T23:54:20,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731974057870, region=557941f9166b64eb2ec3d166e365717f, REOPEN/MOVE in 521 msec 2024-11-18T23:54:20,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T23:54:20,269 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:54:20,269 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:20,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:20,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:20,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:54:20,275 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T23:54:20,275 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=520236456, stopped=false 2024-11-18T23:54:20,275 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9c732c6543ff,41843,1731974054954 2024-11-18T23:54:20,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:20,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:20,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:20,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:20,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:20,278 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T23:54:20,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:20,279 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:20,279 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:20,279 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:54:20,279 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:20,279 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:20,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:20,280 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,46031,1731974055714' ***** 2024-11-18T23:54:20,280 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T23:54:20,280 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,33549,1731974055848' ***** 2024-11-18T23:54:20,280 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T23:54:20,280 INFO [RS:0;9c732c6543ff:46031 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T23:54:20,280 INFO [RS:0;9c732c6543ff:46031 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/bucket.persistence 2024-11-18T23:54:20,280 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T23:54:20,280 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T23:54:20,281 INFO [RS:1;9c732c6543ff:33549 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T23:54:20,281 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T23:54:20,281 INFO [RS:0;9c732c6543ff:46031 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T23:54:20,281 INFO [RS:1;9c732c6543ff:33549 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/bucket.persistence 2024-11-18T23:54:20,281 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T23:54:20,281 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T23:54:20,281 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T23:54:20,281 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T23:54:20,281 INFO [RS:1;9c732c6543ff:33549 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T23:54:20,281 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T23:54:20,291 DEBUG [RS:0;9c732c6543ff:46031 {}] bucket.BucketCache(1487): Thread RS:0;9c732c6543ff:46031 started persisting bucket cache to file 2024-11-18T23:54:20,291 DEBUG [RS:0;9c732c6543ff:46031 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T23:54:20,291 DEBUG [RS:0;9c732c6543ff:46031 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T23:54:20,291 DEBUG [RS:1;9c732c6543ff:33549 {}] bucket.BucketCache(1487): Thread RS:1;9c732c6543ff:33549 started persisting bucket cache to file 2024-11-18T23:54:20,292 DEBUG [RS:1;9c732c6543ff:33549 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T23:54:20,292 DEBUG [RS:1;9c732c6543ff:33549 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T23:54:20,295 DEBUG [RS:0;9c732c6543ff:46031 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T23:54:20,295 DEBUG [RS:0;9c732c6543ff:46031 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731974060291 2024-11-18T23:54:20,295 DEBUG [RS:0;9c732c6543ff:46031 {}] bucket.BucketCache(1509): Thread RS:0;9c732c6543ff:46031 finished persisting bucket cache to file, renaming 2024-11-18T23:54:20,295 DEBUG [RS:1;9c732c6543ff:33549 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T23:54:20,295 DEBUG [RS:1;9c732c6543ff:33549 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731974060291 2024-11-18T23:54:20,295 DEBUG [RS:1;9c732c6543ff:33549 {}] bucket.BucketCache(1509): Thread RS:1;9c732c6543ff:33549 finished persisting bucket cache to file, renaming 2024-11-18T23:54:20,296 WARN [RS:1;9c732c6543ff:33549 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-18T23:54:20,295 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T23:54:20,296 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T23:54:20,296 INFO [RS:1;9c732c6543ff:33549 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T23:54:20,296 INFO [RS:0;9c732c6543ff:46031 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T23:54:20,296 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T23:54:20,296 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T23:54:20,296 INFO [RS:0;9c732c6543ff:46031 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T23:54:20,296 INFO [RS:1;9c732c6543ff:33549 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T23:54:20,296 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,33549,1731974055848 2024-11-18T23:54:20,296 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(3091): Received CLOSE for 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,296 INFO [RS:1;9c732c6543ff:33549 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:54:20,296 INFO [RS:1;9c732c6543ff:33549 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9c732c6543ff:33549. 2024-11-18T23:54:20,296 DEBUG [RS:1;9c732c6543ff:33549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:20,297 DEBUG [RS:1;9c732c6543ff:33549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:20,297 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,33549,1731974055848; all regions closed. 2024-11-18T23:54:20,297 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,46031,1731974055714 2024-11-18T23:54:20,297 INFO [RS:0;9c732c6543ff:46031 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:54:20,297 INFO [RS:0;9c732c6543ff:46031 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9c732c6543ff:46031. 2024-11-18T23:54:20,297 DEBUG [RS:0;9c732c6543ff:46031 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:20,297 DEBUG [RS:0;9c732c6543ff:46031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:20,298 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T23:54:20,298 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T23:54:20,298 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T23:54:20,298 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T23:54:20,298 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T23:54:20,298 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 557941f9166b64eb2ec3d166e365717f, disabling compactions & flushes 2024-11-18T23:54:20,298 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:20,298 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:20,298 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. after waiting 0 ms 2024-11-18T23:54:20,298 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:20,298 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-18T23:54:20,298 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T23:54:20,299 DEBUG [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(1325): Online Regions={557941f9166b64eb2ec3d166e365717f=testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f., 1588230740=hbase:meta,,1.1588230740} 2024-11-18T23:54:20,299 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T23:54:20,299 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T23:54:20,299 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T23:54:20,299 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T23:54:20,299 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T23:54:20,299 DEBUG [StoreFileCloser-557941f9166b64eb2ec3d166e365717f-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/family/fd16d73d659549f692fa0d337a88292f 2024-11-18T23:54:20,299 DEBUG [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 557941f9166b64eb2ec3d166e365717f 2024-11-18T23:54:20,299 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-18T23:54:20,301 DEBUG [RS:1;9c732c6543ff:33549 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs 2024-11-18T23:54:20,302 INFO [RS:1;9c732c6543ff:33549 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9c732c6543ff%2C33549%2C1731974055848:(num 1731974057166) 2024-11-18T23:54:20,302 DEBUG [RS:1;9c732c6543ff:33549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:20,302 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:20,302 INFO [RS:1;9c732c6543ff:33549 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:54:20,302 INFO [RS:1;9c732c6543ff:33549 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T23:54:20,302 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T23:54:20,303 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T23:54:20,303 INFO [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:54:20,303 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T23:54:20,303 INFO [RS:1;9c732c6543ff:33549 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:54:20,303 INFO [RS:1;9c732c6543ff:33549 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33549 2024-11-18T23:54:20,304 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnRegionMove1731974057870/557941f9166b64eb2ec3d166e365717f/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-18T23:54:20,305 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:20,305 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 557941f9166b64eb2ec3d166e365717f: Waiting for close lock at 1731974060297Running coprocessor pre-close hooks at 1731974060298 (+1 ms)Disabling compacts and flushes for region at 1731974060298Disabling writes for close at 1731974060298Writing region close event to WAL at 1731974060299 (+1 ms)Running coprocessor post-close hooks at 1731974060305 (+6 ms)Closed at 1731974060305 2024-11-18T23:54:20,305 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f. 2024-11-18T23:54:20,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,33549,1731974055848 2024-11-18T23:54:20,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:54:20,308 INFO [RS:1;9c732c6543ff:33549 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:54:20,310 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,33549,1731974055848] 2024-11-18T23:54:20,311 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1f61692ef8c241db887132c2a1704da8 2024-11-18T23:54:20,312 INFO [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:20,312 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,33549,1731974055848 already deleted, retry=false 2024-11-18T23:54:20,312 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,33549,1731974055848 expired; onlineServers=1 2024-11-18T23:54:20,318 INFO [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:20,328 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/info/1f61692ef8c241db887132c2a1704da8 is 205, key is testBlockEvictionOnRegionMove1731974057870,,1731974057885.557941f9166b64eb2ec3d166e365717f./info:regioninfo/1731974060230/Put/seqid=0 2024-11-18T23:54:20,329 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/info/1f61692ef8c241db887132c2a1704da8 2024-11-18T23:54:20,339 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a2989795dddf40c189d1c0f5d5430631 2024-11-18T23:54:20,353 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/ns/a2989795dddf40c189d1c0f5d5430631 is 43, key is default/ns:d/1731974057682/Put/seqid=0 2024-11-18T23:54:20,354 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/ns/a2989795dddf40c189d1c0f5d5430631 2024-11-18T23:54:20,363 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ff73af2e03344d64a216320170f4627e 2024-11-18T23:54:20,377 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/table/ff73af2e03344d64a216320170f4627e is 78, key is testBlockEvictionOnRegionMove1731974057870/table:state/1731974058294/Put/seqid=0 2024-11-18T23:54:20,378 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/table/ff73af2e03344d64a216320170f4627e 2024-11-18T23:54:20,380 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/info/1f61692ef8c241db887132c2a1704da8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/1f61692ef8c241db887132c2a1704da8 2024-11-18T23:54:20,382 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/1f61692ef8c241db887132c2a1704da8, delay=907 ms 2024-11-18T23:54:20,383 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/1f61692ef8c241db887132c2a1704da8, entries=15, sequenceid=15, filesize=7.6 K 2024-11-18T23:54:20,383 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/ns/a2989795dddf40c189d1c0f5d5430631 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/ns/a2989795dddf40c189d1c0f5d5430631 2024-11-18T23:54:20,385 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/ns/a2989795dddf40c189d1c0f5d5430631, delay=951 ms 2024-11-18T23:54:20,386 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/ns/a2989795dddf40c189d1c0f5d5430631, entries=2, sequenceid=15, filesize=5.0 K 2024-11-18T23:54:20,386 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/table/ff73af2e03344d64a216320170f4627e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/table/ff73af2e03344d64a216320170f4627e 2024-11-18T23:54:20,388 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/table/ff73af2e03344d64a216320170f4627e, delay=980 ms 2024-11-18T23:54:20,389 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/table/ff73af2e03344d64a216320170f4627e, entries=2, sequenceid=15, filesize=5.3 K 2024-11-18T23:54:20,389 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false 2024-11-18T23:54:20,390 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T23:54:20,391 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/1f61692ef8c241db887132c2a1704da8 2024-11-18T23:54:20,392 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/ns/a2989795dddf40c189d1c0f5d5430631 2024-11-18T23:54:20,393 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/table/ff73af2e03344d64a216320170f4627e 2024-11-18T23:54:20,398 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-18T23:54:20,398 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T23:54:20,399 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T23:54:20,399 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731974060298Running coprocessor pre-close hooks at 1731974060299 (+1 ms)Disabling compacts and flushes for region at 1731974060299Disabling writes for close at 1731974060299Obtaining lock to block concurrent updates at 1731974060299Preparing flush snapshotting stores in 1588230740 at 1731974060299Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731974060300 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731974060300Flushing 1588230740/info: creating writer at 1731974060300Flushing 1588230740/info: appending metadata at 1731974060328 (+28 ms)Flushing 1588230740/info: closing flushed file at 1731974060328Flushing 1588230740/ns: creating writer at 1731974060332 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731974060352 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731974060352Flushing 1588230740/table: creating writer at 1731974060356 (+4 ms)Flushing 1588230740/table: appending metadata at 1731974060376 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731974060377 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2550645d: reopening flushed file at 1731974060380 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a7b3a91: reopening flushed file at 1731974060383 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@357f0344: reopening flushed file at 1731974060386 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false at 1731974060390 (+4 ms)Writing region close event to WAL at 1731974060394 (+4 ms)Running coprocessor post-close hooks at 1731974060398 (+4 ms)Closed at 1731974060399 (+1 ms) 2024-11-18T23:54:20,399 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T23:54:20,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:20,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33549-0x1010845b7b00002, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:20,411 INFO [RS:1;9c732c6543ff:33549 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:54:20,411 INFO [RS:1;9c732c6543ff:33549 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,33549,1731974055848; zookeeper connection closed. 2024-11-18T23:54:20,412 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@ab88d34 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@ab88d34 2024-11-18T23:54:20,499 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,46031,1731974055714; all regions closed. 2024-11-18T23:54:20,502 DEBUG [RS:0;9c732c6543ff:46031 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs 2024-11-18T23:54:20,502 INFO [RS:0;9c732c6543ff:46031 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9c732c6543ff%2C46031%2C1731974055714.meta:.meta(num 1731974057552) 2024-11-18T23:54:20,504 DEBUG [RS:0;9c732c6543ff:46031 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs 2024-11-18T23:54:20,504 INFO [RS:0;9c732c6543ff:46031 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9c732c6543ff%2C46031%2C1731974055714:(num 1731974057167) 2024-11-18T23:54:20,504 DEBUG [RS:0;9c732c6543ff:46031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:20,504 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:20,504 INFO [RS:0;9c732c6543ff:46031 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:54:20,504 INFO [RS:0;9c732c6543ff:46031 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T23:54:20,504 INFO [RS:0;9c732c6543ff:46031 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:54:20,504 INFO [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:54:20,505 INFO [RS:0;9c732c6543ff:46031 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46031 2024-11-18T23:54:20,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:54:20,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,46031,1731974055714 2024-11-18T23:54:20,507 INFO [RS:0;9c732c6543ff:46031 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:54:20,509 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,46031,1731974055714] 2024-11-18T23:54:20,510 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,46031,1731974055714 already deleted, retry=false 2024-11-18T23:54:20,510 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,46031,1731974055714 expired; onlineServers=0 2024-11-18T23:54:20,510 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9c732c6543ff,41843,1731974054954' ***** 2024-11-18T23:54:20,510 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T23:54:20,510 INFO [M:0;9c732c6543ff:41843 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:54:20,511 INFO [M:0;9c732c6543ff:41843 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:54:20,511 DEBUG [M:0;9c732c6543ff:41843 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T23:54:20,511 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T23:54:20,511 DEBUG [M:0;9c732c6543ff:41843 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T23:54:20,511 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974056825 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974056825,5,FailOnTimeoutGroup] 2024-11-18T23:54:20,511 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974056824 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974056824,5,FailOnTimeoutGroup] 2024-11-18T23:54:20,511 INFO [M:0;9c732c6543ff:41843 {}] hbase.ChoreService(370): Chore service for: master/9c732c6543ff:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T23:54:20,511 INFO [M:0;9c732c6543ff:41843 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:54:20,512 DEBUG [M:0;9c732c6543ff:41843 {}] master.HMaster(1795): Stopping service threads 2024-11-18T23:54:20,512 INFO [M:0;9c732c6543ff:41843 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T23:54:20,512 INFO [M:0;9c732c6543ff:41843 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T23:54:20,512 INFO [M:0;9c732c6543ff:41843 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T23:54:20,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T23:54:20,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:20,513 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T23:54:20,513 DEBUG [M:0;9c732c6543ff:41843 {}] zookeeper.ZKUtil(347): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T23:54:20,513 WARN [M:0;9c732c6543ff:41843 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T23:54:20,514 INFO [M:0;9c732c6543ff:41843 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/.lastflushedseqids 2024-11-18T23:54:20,529 INFO [M:0;9c732c6543ff:41843 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T23:54:20,529 INFO [M:0;9c732c6543ff:41843 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T23:54:20,530 DEBUG [M:0;9c732c6543ff:41843 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T23:54:20,530 INFO [M:0;9c732c6543ff:41843 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:20,530 DEBUG [M:0;9c732c6543ff:41843 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:20,530 DEBUG [M:0;9c732c6543ff:41843 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T23:54:20,530 DEBUG [M:0;9c732c6543ff:41843 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:20,530 INFO [M:0;9c732c6543ff:41843 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-18T23:54:20,541 DEBUG [M:0;9c732c6543ff:41843 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d7f3451f429c45dea3d6da09c0921a2e 2024-11-18T23:54:20,556 DEBUG [M:0;9c732c6543ff:41843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d7f3451f429c45dea3d6da09c0921a2e is 82, key is hbase:meta,,1/info:regioninfo/1731974057616/Put/seqid=0 2024-11-18T23:54:20,556 INFO [M:0;9c732c6543ff:41843 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d7f3451f429c45dea3d6da09c0921a2e 2024-11-18T23:54:20,565 DEBUG [M:0;9c732c6543ff:41843 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bef39561f005436886242c7ba07cc981 2024-11-18T23:54:20,580 DEBUG [M:0;9c732c6543ff:41843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bef39561f005436886242c7ba07cc981 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731974058299/Put/seqid=0 2024-11-18T23:54:20,580 INFO [M:0;9c732c6543ff:41843 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bef39561f005436886242c7ba07cc981 2024-11-18T23:54:20,589 DEBUG [M:0;9c732c6543ff:41843 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 523fd5c9631c4e4c927661fadd5da4d8 2024-11-18T23:54:20,604 DEBUG [M:0;9c732c6543ff:41843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/523fd5c9631c4e4c927661fadd5da4d8 is 69, key is 9c732c6543ff,33549,1731974055848/rs:state/1731974056935/Put/seqid=0 2024-11-18T23:54:20,605 INFO [M:0;9c732c6543ff:41843 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/523fd5c9631c4e4c927661fadd5da4d8 2024-11-18T23:54:20,607 DEBUG [M:0;9c732c6543ff:41843 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d7f3451f429c45dea3d6da09c0921a2e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d7f3451f429c45dea3d6da09c0921a2e 2024-11-18T23:54:20,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:20,609 INFO [RS:0;9c732c6543ff:46031 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:54:20,609 INFO [M:0;9c732c6543ff:41843 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d7f3451f429c45dea3d6da09c0921a2e, entries=8, sequenceid=84, filesize=5.5 K 2024-11-18T23:54:20,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46031-0x1010845b7b00001, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:20,609 INFO [RS:0;9c732c6543ff:46031 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,46031,1731974055714; zookeeper connection closed. 2024-11-18T23:54:20,609 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a529e1b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a529e1b 2024-11-18T23:54:20,610 DEBUG [M:0;9c732c6543ff:41843 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bef39561f005436886242c7ba07cc981 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bef39561f005436886242c7ba07cc981 2024-11-18T23:54:20,610 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-18T23:54:20,612 INFO [M:0;9c732c6543ff:41843 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bef39561f005436886242c7ba07cc981, entries=9, sequenceid=84, filesize=6.5 K 2024-11-18T23:54:20,612 DEBUG [M:0;9c732c6543ff:41843 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/523fd5c9631c4e4c927661fadd5da4d8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/523fd5c9631c4e4c927661fadd5da4d8 2024-11-18T23:54:20,615 INFO [M:0;9c732c6543ff:41843 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/523fd5c9631c4e4c927661fadd5da4d8, entries=2, sequenceid=84, filesize=5.1 K 2024-11-18T23:54:20,616 INFO [M:0;9c732c6543ff:41843 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=84, compaction requested=false 2024-11-18T23:54:20,618 INFO [M:0;9c732c6543ff:41843 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:20,618 DEBUG [M:0;9c732c6543ff:41843 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731974060529Disabling compacts and flushes for region at 1731974060529Disabling writes for close at 1731974060530 (+1 ms)Obtaining lock to block concurrent updates at 1731974060530Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731974060530Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731974060531 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731974060531Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731974060531Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731974060555 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731974060555Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731974060558 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731974060579 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731974060579Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731974060582 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731974060604 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731974060604Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22f62144: reopening flushed file at 1731974060607 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c4f835b: reopening flushed file at 1731974060609 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12f03042: reopening flushed file at 1731974060612 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=84, compaction requested=false at 1731974060616 (+4 ms)Writing region close event to WAL at 1731974060617 (+1 ms)Closed at 1731974060618 (+1 ms) 2024-11-18T23:54:20,619 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:54:20,620 INFO [M:0;9c732c6543ff:41843 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T23:54:20,620 INFO [M:0;9c732c6543ff:41843 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41843 2024-11-18T23:54:20,620 INFO [M:0;9c732c6543ff:41843 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:54:20,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:20,722 INFO [M:0;9c732c6543ff:41843 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:54:20,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41843-0x1010845b7b00000, quorum=127.0.0.1:59958, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:20,727 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T23:54:20,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T23:54:20,748 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:20,757 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731974060217 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731974060385 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/9c732c6543ff:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731974060382 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/9c732c6543ff:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/9c732c6543ff:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731974058691 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/9c732c6543ff:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=106 (was 99) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4891 (was 5234) 2024-11-18T23:54:20,763 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=106, ProcessCount=11, AvailableMemoryMB=4890 2024-11-18T23:54:20,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/hadoop-log-dir so I do NOT create it in target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227 2024-11-18T23:54:20,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/hadoop-tmp-dir so I do NOT create it in target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227 2024-11-18T23:54:20,764 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:20,767 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/cluster_86d39c44-2521-5283-7831-fa60551143c9, deleteOnExit=true 2024-11-18T23:54:20,770 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/cluster_86d39c44-2521-5283-7831-fa60551143c9/zookeeper_0, clientPort=52910, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/cluster_86d39c44-2521-5283-7831-fa60551143c9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/cluster_86d39c44-2521-5283-7831-fa60551143c9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T23:54:20,771 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52910 2024-11-18T23:54:20,783 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d with version=8 2024-11-18T23:54:20,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/hbase-staging 2024-11-18T23:54:20,786 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9c732c6543ff:0 server-side Connection retries=45 2024-11-18T23:54:20,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:20,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:20,786 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:54:20,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:20,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:54:20,786 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T23:54:20,787 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:54:20,787 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38331 2024-11-18T23:54:20,789 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38331 connecting to ZooKeeper ensemble=127.0.0.1:52910 2024-11-18T23:54:20,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:383310x0, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:54:20,795 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38331-0x1010845cf8b0000 connected 2024-11-18T23:54:20,814 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:20,814 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:20,815 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:20,815 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d, hbase.cluster.distributed=false 2024-11-18T23:54:20,817 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:54:20,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38331 2024-11-18T23:54:20,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38331 2024-11-18T23:54:20,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38331 2024-11-18T23:54:20,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38331 2024-11-18T23:54:20,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38331 2024-11-18T23:54:20,834 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45 2024-11-18T23:54:20,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:20,835 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:20,835 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:54:20,835 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:20,835 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:54:20,835 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T23:54:20,835 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:54:20,836 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38947 2024-11-18T23:54:20,837 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38947 connecting to ZooKeeper ensemble=127.0.0.1:52910 2024-11-18T23:54:20,837 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:20,838 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:20,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:389470x0, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:54:20,841 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38947-0x1010845cf8b0001 connected 2024-11-18T23:54:20,841 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:20,842 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T23:54:20,843 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.cache 2024-11-18T23:54:20,843 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T23:54:20,843 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T23:54:20,843 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T23:54:20,843 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T23:54:20,843 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T23:54:20,843 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T23:54:20,844 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T23:54:20,844 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T23:54:20,844 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.persistence is complete. 2024-11-18T23:54:20,844 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T23:54:20,846 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:54:20,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38947 2024-11-18T23:54:20,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38947 2024-11-18T23:54:20,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38947 2024-11-18T23:54:20,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38947 2024-11-18T23:54:20,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38947 2024-11-18T23:54:20,867 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45 2024-11-18T23:54:20,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:20,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:20,868 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:54:20,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:20,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:54:20,868 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T23:54:20,868 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:54:20,869 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43617 2024-11-18T23:54:20,871 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43617 connecting to ZooKeeper ensemble=127.0.0.1:52910 2024-11-18T23:54:20,872 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:20,872 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:20,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436170x0, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:54:20,876 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43617-0x1010845cf8b0002 connected 2024-11-18T23:54:20,876 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:20,876 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T23:54:20,877 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.cache 2024-11-18T23:54:20,877 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T23:54:20,877 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T23:54:20,877 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T23:54:20,878 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T23:54:20,878 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T23:54:20,878 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T23:54:20,878 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T23:54:20,878 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T23:54:20,878 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.persistence is complete. 2024-11-18T23:54:20,879 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T23:54:20,880 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:54:20,885 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43617 2024-11-18T23:54:20,885 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43617 2024-11-18T23:54:20,886 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43617 2024-11-18T23:54:20,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43617 2024-11-18T23:54:20,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43617 2024-11-18T23:54:20,905 DEBUG [M:0;9c732c6543ff:38331 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9c732c6543ff:38331 2024-11-18T23:54:20,906 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9c732c6543ff,38331,1731974060785 2024-11-18T23:54:20,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:20,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:20,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:20,909 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9c732c6543ff,38331,1731974060785 2024-11-18T23:54:20,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T23:54:20,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T23:54:20,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:20,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:20,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:20,913 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T23:54:20,914 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9c732c6543ff,38331,1731974060785 from backup master directory 2024-11-18T23:54:20,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:20,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:20,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9c732c6543ff,38331,1731974060785 2024-11-18T23:54:20,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:54:20,915 WARN [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:54:20,915 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9c732c6543ff,38331,1731974060785 2024-11-18T23:54:20,917 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/hbase.id] with ID: 02b9c948-49eb-48f0-922c-12740ac403a4 2024-11-18T23:54:20,917 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/.tmp/hbase.id 2024-11-18T23:54:20,926 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/hbase.id] 2024-11-18T23:54:20,966 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:20,967 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T23:54:20,968 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-18T23:54:20,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:20,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:20,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:20,976 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:54:20,978 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T23:54:20,978 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:20,998 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store 2024-11-18T23:54:21,005 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:21,005 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T23:54:21,005 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:21,005 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:21,005 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T23:54:21,005 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:21,005 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:21,005 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731974061005Disabling compacts and flushes for region at 1731974061005Disabling writes for close at 1731974061005Writing region close event to WAL at 1731974061005Closed at 1731974061005 2024-11-18T23:54:21,005 WARN [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/.initializing 2024-11-18T23:54:21,005 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/WALs/9c732c6543ff,38331,1731974060785 2024-11-18T23:54:21,011 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:54:21,015 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9c732c6543ff%2C38331%2C1731974060785, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/WALs/9c732c6543ff,38331,1731974060785, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/oldWALs, maxLogs=10 2024-11-18T23:54:21,033 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/WALs/9c732c6543ff,38331,1731974060785/9c732c6543ff%2C38331%2C1731974060785.1731974061015 2024-11-18T23:54:21,034 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:21,034 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:21,034 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:21,034 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,034 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,035 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,041 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T23:54:21,041 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:21,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T23:54:21,046 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:21,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T23:54:21,052 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:21,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T23:54:21,056 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:21,056 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,057 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,057 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,057 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,057 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,058 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:54:21,058 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:54:21,064 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:54:21,065 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60983758, jitterRate=-0.09127119183540344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:54:21,066 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731974061034Initializing all the Stores at 1731974061034Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974061035 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974061035Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974061035Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974061035Cleaning up temporary data from old regions at 1731974061058 (+23 ms)Region opened successfully at 1731974061065 (+7 ms) 2024-11-18T23:54:21,066 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T23:54:21,070 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1550ec3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0 2024-11-18T23:54:21,071 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T23:54:21,072 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T23:54:21,072 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T23:54:21,072 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T23:54:21,072 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T23:54:21,073 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-18T23:54:21,073 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T23:54:21,075 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T23:54:21,076 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T23:54:21,077 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T23:54:21,078 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T23:54:21,078 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T23:54:21,080 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T23:54:21,080 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T23:54:21,081 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T23:54:21,082 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T23:54:21,084 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T23:54:21,085 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T23:54:21,087 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T23:54:21,088 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T23:54:21,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:21,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:21,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:21,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,091 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9c732c6543ff,38331,1731974060785, sessionid=0x1010845cf8b0000, setting cluster-up flag (Was=false) 2024-11-18T23:54:21,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,099 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T23:54:21,100 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,38331,1731974060785 2024-11-18T23:54:21,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,108 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T23:54:21,110 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9c732c6543ff,38331,1731974060785 2024-11-18T23:54:21,110 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T23:54:21,113 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:21,113 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T23:54:21,113 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T23:54:21,114 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9c732c6543ff,38331,1731974060785 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T23:54:21,116 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:21,116 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:21,116 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:21,116 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9c732c6543ff:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:54:21,116 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9c732c6543ff:0, corePoolSize=10, maxPoolSize=10 2024-11-18T23:54:21,116 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,116 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:54:21,117 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,117 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731974091117 2024-11-18T23:54:21,118 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T23:54:21,118 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T23:54:21,118 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T23:54:21,118 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T23:54:21,118 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T23:54:21,118 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T23:54:21,118 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:21,118 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T23:54:21,119 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,119 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T23:54:21,121 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,121 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T23:54:21,122 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T23:54:21,122 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T23:54:21,123 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T23:54:21,123 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T23:54:21,124 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974061123,5,FailOnTimeoutGroup] 2024-11-18T23:54:21,125 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974061124,5,FailOnTimeoutGroup] 2024-11-18T23:54:21,125 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,125 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T23:54:21,125 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,126 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,138 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T23:54:21,138 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d 2024-11-18T23:54:21,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:21,145 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T23:54:21,149 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T23:54:21,149 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:21,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T23:54:21,153 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T23:54:21,153 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:21,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T23:54:21,156 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T23:54:21,157 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:21,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T23:54:21,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T23:54:21,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:21,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T23:54:21,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740 2024-11-18T23:54:21,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740 2024-11-18T23:54:21,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T23:54:21,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T23:54:21,162 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:54:21,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T23:54:21,168 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:54:21,168 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73058095, jitterRate=0.08865045011043549}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:54:21,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731974061145Initializing all the Stores at 1731974061145Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974061145Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974061145Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974061145Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974061145Cleaning up temporary data from old regions at 1731974061161 (+16 ms)Region opened successfully at 1731974061169 (+8 ms) 2024-11-18T23:54:21,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T23:54:21,169 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T23:54:21,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T23:54:21,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T23:54:21,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T23:54:21,170 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T23:54:21,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731974061169Disabling compacts and flushes for region at 1731974061169Disabling writes for close at 1731974061169Writing region close event to WAL at 1731974061170 (+1 ms)Closed at 1731974061170 2024-11-18T23:54:21,171 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:21,171 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T23:54:21,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T23:54:21,172 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:54:21,173 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T23:54:21,195 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(746): ClusterId : 02b9c948-49eb-48f0-922c-12740ac403a4 2024-11-18T23:54:21,195 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(746): ClusterId : 02b9c948-49eb-48f0-922c-12740ac403a4 2024-11-18T23:54:21,195 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T23:54:21,195 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T23:54:21,198 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T23:54:21,198 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T23:54:21,198 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T23:54:21,198 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T23:54:21,201 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T23:54:21,201 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T23:54:21,201 DEBUG [RS:1;9c732c6543ff:43617 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13041118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0 2024-11-18T23:54:21,201 DEBUG [RS:0;9c732c6543ff:38947 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636de062, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0 2024-11-18T23:54:21,221 DEBUG [RS:0;9c732c6543ff:38947 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9c732c6543ff:38947 2024-11-18T23:54:21,221 DEBUG [RS:1;9c732c6543ff:43617 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9c732c6543ff:43617 2024-11-18T23:54:21,221 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T23:54:21,221 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T23:54:21,221 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T23:54:21,221 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T23:54:21,221 DEBUG [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T23:54:21,221 DEBUG [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T23:54:21,222 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,38331,1731974060785 with port=38947, startcode=1731974060834 2024-11-18T23:54:21,223 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,38331,1731974060785 with port=43617, startcode=1731974060867 2024-11-18T23:54:21,223 DEBUG [RS:0;9c732c6543ff:38947 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T23:54:21,223 DEBUG [RS:1;9c732c6543ff:43617 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T23:54:21,226 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35445, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T23:54:21,226 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50383, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T23:54:21,226 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38331 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,43617,1731974060867 2024-11-18T23:54:21,226 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38331 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,43617,1731974060867 2024-11-18T23:54:21,228 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38331 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,38947,1731974060834 2024-11-18T23:54:21,228 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38331 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,38947,1731974060834 2024-11-18T23:54:21,228 DEBUG [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d 2024-11-18T23:54:21,228 DEBUG [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T23:54:21,228 DEBUG [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T23:54:21,230 DEBUG [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d 2024-11-18T23:54:21,230 DEBUG [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T23:54:21,230 DEBUG [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T23:54:21,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:54:21,235 DEBUG [RS:1;9c732c6543ff:43617 {}] zookeeper.ZKUtil(111): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,43617,1731974060867 2024-11-18T23:54:21,235 WARN [RS:1;9c732c6543ff:43617 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:54:21,235 INFO [RS:1;9c732c6543ff:43617 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:21,235 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,43617,1731974060867] 2024-11-18T23:54:21,235 DEBUG [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,43617,1731974060867 2024-11-18T23:54:21,235 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,38947,1731974060834] 2024-11-18T23:54:21,235 DEBUG [RS:0;9c732c6543ff:38947 {}] zookeeper.ZKUtil(111): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,38947,1731974060834 2024-11-18T23:54:21,235 WARN [RS:0;9c732c6543ff:38947 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:54:21,235 INFO [RS:0;9c732c6543ff:38947 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:21,235 DEBUG [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,38947,1731974060834 2024-11-18T23:54:21,240 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T23:54:21,243 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T23:54:21,243 INFO [RS:1;9c732c6543ff:43617 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:54:21,243 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T23:54:21,243 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,244 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T23:54:21,244 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T23:54:21,245 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,245 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,245 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,245 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,245 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,245 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,245 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:54:21,246 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,246 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T23:54:21,246 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,246 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,246 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,246 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,246 INFO [RS:0;9c732c6543ff:38947 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:54:21,246 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,246 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,246 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:21,246 DEBUG [RS:1;9c732c6543ff:43617 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:21,246 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T23:54:21,247 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,247 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T23:54:21,247 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,247 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,247 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,247 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,247 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,247 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,43617,1731974060867-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:54:21,247 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,247 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:21,248 DEBUG [RS:0;9c732c6543ff:38947 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:21,249 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,249 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,249 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,249 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,249 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,249 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,38947,1731974060834-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:54:21,270 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T23:54:21,271 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,43617,1731974060867-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,271 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,271 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.Replication(171): 9c732c6543ff,43617,1731974060867 started 2024-11-18T23:54:21,271 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T23:54:21,272 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,38947,1731974060834-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,272 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,272 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.Replication(171): 9c732c6543ff,38947,1731974060834 started 2024-11-18T23:54:21,289 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,289 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,43617,1731974060867, RpcServer on 9c732c6543ff/172.17.0.2:43617, sessionid=0x1010845cf8b0002 2024-11-18T23:54:21,289 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T23:54:21,289 DEBUG [RS:1;9c732c6543ff:43617 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,43617,1731974060867 2024-11-18T23:54:21,289 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,43617,1731974060867' 2024-11-18T23:54:21,289 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T23:54:21,290 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,290 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,38947,1731974060834, RpcServer on 9c732c6543ff/172.17.0.2:38947, sessionid=0x1010845cf8b0001 2024-11-18T23:54:21,290 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T23:54:21,290 DEBUG [RS:0;9c732c6543ff:38947 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,38947,1731974060834 2024-11-18T23:54:21,290 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,38947,1731974060834' 2024-11-18T23:54:21,290 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T23:54:21,290 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T23:54:21,291 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T23:54:21,291 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T23:54:21,291 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T23:54:21,291 DEBUG [RS:1;9c732c6543ff:43617 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,43617,1731974060867 2024-11-18T23:54:21,291 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,43617,1731974060867' 2024-11-18T23:54:21,291 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T23:54:21,291 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T23:54:21,291 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T23:54:21,291 DEBUG [RS:0;9c732c6543ff:38947 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,38947,1731974060834 2024-11-18T23:54:21,291 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,38947,1731974060834' 2024-11-18T23:54:21,291 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T23:54:21,291 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T23:54:21,292 DEBUG [RS:1;9c732c6543ff:43617 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T23:54:21,292 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T23:54:21,292 INFO [RS:1;9c732c6543ff:43617 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T23:54:21,292 INFO [RS:1;9c732c6543ff:43617 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T23:54:21,292 DEBUG [RS:0;9c732c6543ff:38947 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T23:54:21,293 INFO [RS:0;9c732c6543ff:38947 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T23:54:21,293 INFO [RS:0;9c732c6543ff:38947 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T23:54:21,323 WARN [9c732c6543ff:38331 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T23:54:21,393 INFO [RS:1;9c732c6543ff:43617 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:54:21,393 INFO [RS:0;9c732c6543ff:38947 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:54:21,393 INFO [RS:1;9c732c6543ff:43617 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9c732c6543ff%2C43617%2C1731974060867, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,43617,1731974060867, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs, maxLogs=55 2024-11-18T23:54:21,394 INFO [RS:0;9c732c6543ff:38947 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9c732c6543ff%2C38947%2C1731974060834, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,38947,1731974060834, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs, maxLogs=55 2024-11-18T23:54:21,413 INFO [RS:0;9c732c6543ff:38947 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,38947,1731974060834/9c732c6543ff%2C38947%2C1731974060834.1731974061395 2024-11-18T23:54:21,413 DEBUG [RS:0;9c732c6543ff:38947 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:21,414 INFO [RS:1;9c732c6543ff:43617 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,43617,1731974060867/9c732c6543ff%2C43617%2C1731974060867.1731974061395 2024-11-18T23:54:21,415 DEBUG [RS:1;9c732c6543ff:43617 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:21,574 DEBUG [9c732c6543ff:38331 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-18T23:54:21,574 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(204): Hosts are {9c732c6543ff=0} racks are {/default-rack=0} 2024-11-18T23:54:21,575 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T23:54:21,575 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T23:54:21,575 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T23:54:21,575 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T23:54:21,575 INFO [9c732c6543ff:38331 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T23:54:21,575 INFO [9c732c6543ff:38331 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T23:54:21,575 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T23:54:21,575 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9c732c6543ff,43617,1731974060867 2024-11-18T23:54:21,576 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,43617,1731974060867, state=OPENING 2024-11-18T23:54:21,578 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T23:54:21,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:21,581 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:21,581 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:54:21,581 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:21,581 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:21,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,43617,1731974060867}] 2024-11-18T23:54:21,734 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:54:21,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33697, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:54:21,738 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T23:54:21,738 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:21,739 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T23:54:21,739 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9c732c6543ff%2C43617%2C1731974060867.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,43617,1731974060867, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs, maxLogs=55 2024-11-18T23:54:21,758 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,43617,1731974060867/9c732c6543ff%2C43617%2C1731974060867.meta.1731974061740.meta 2024-11-18T23:54:21,759 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:21,759 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:21,759 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T23:54:21,759 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T23:54:21,760 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T23:54:21,760 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T23:54:21,760 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:21,760 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T23:54:21,760 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T23:54:21,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T23:54:21,762 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T23:54:21,762 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:21,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T23:54:21,763 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T23:54:21,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:21,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T23:54:21,763 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T23:54:21,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:21,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T23:54:21,764 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T23:54:21,764 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:21,764 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T23:54:21,765 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740 2024-11-18T23:54:21,765 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740 2024-11-18T23:54:21,766 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T23:54:21,766 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T23:54:21,766 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:54:21,767 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T23:54:21,767 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66458391, jitterRate=-0.009692803025245667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:54:21,767 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T23:54:21,768 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731974061760Writing region info on filesystem at 1731974061760Initializing all the Stores at 1731974061760Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974061761 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974061761Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974061761Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974061761Cleaning up temporary data from old regions at 1731974061766 (+5 ms)Running coprocessor post-open hooks at 1731974061767 (+1 ms)Region opened successfully at 1731974061768 (+1 ms) 2024-11-18T23:54:21,769 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731974061733 2024-11-18T23:54:21,772 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T23:54:21,772 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T23:54:21,773 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,43617,1731974060867 2024-11-18T23:54:21,773 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,43617,1731974060867, state=OPEN 2024-11-18T23:54:21,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:21,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:21,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:21,775 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:21,775 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:21,775 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:21,775 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9c732c6543ff,43617,1731974060867 2024-11-18T23:54:21,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T23:54:21,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,43617,1731974060867 in 195 msec 2024-11-18T23:54:21,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T23:54:21,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 607 msec 2024-11-18T23:54:21,781 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:54:21,781 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T23:54:21,784 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:54:21,785 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,43617,1731974060867, seqNum=-1] 2024-11-18T23:54:21,785 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:54:21,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47341, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:54:21,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 679 msec 2024-11-18T23:54:21,792 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731974061792, completionTime=-1 2024-11-18T23:54:21,792 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-18T23:54:21,792 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T23:54:21,794 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T23:54:21,794 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731974121794 2024-11-18T23:54:21,794 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731974181794 2024-11-18T23:54:21,795 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-18T23:54:21,795 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,38331,1731974060785-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,795 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,38331,1731974060785-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,795 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,38331,1731974060785-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,796 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9c732c6543ff:38331, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,796 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,796 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,798 DEBUG [master/9c732c6543ff:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T23:54:21,801 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.886sec 2024-11-18T23:54:21,801 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T23:54:21,801 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T23:54:21,801 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T23:54:21,801 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T23:54:21,801 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T23:54:21,801 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,38331,1731974060785-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:54:21,801 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,38331,1731974060785-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T23:54:21,802 DEBUG [master/9c732c6543ff:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T23:54:21,802 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T23:54:21,803 INFO [master/9c732c6543ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,38331,1731974060785-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:21,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219aa7ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:54:21,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9c732c6543ff,38331,-1 for getting cluster id 2024-11-18T23:54:21,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:54:21,897 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '02b9c948-49eb-48f0-922c-12740ac403a4' 2024-11-18T23:54:21,898 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:54:21,898 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "02b9c948-49eb-48f0-922c-12740ac403a4" 2024-11-18T23:54:21,899 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a88e976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:54:21,899 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9c732c6543ff,38331,-1] 2024-11-18T23:54:21,899 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:54:21,899 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:21,901 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:54:21,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f34b262, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:54:21,903 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:54:21,904 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,43617,1731974060867, seqNum=-1] 2024-11-18T23:54:21,905 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:54:21,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:54:21,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9c732c6543ff,38331,1731974060785 2024-11-18T23:54:21,909 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T23:54:21,911 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 9c732c6543ff,38331,1731974060785 2024-11-18T23:54:21,911 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@194bcc11 2024-11-18T23:54:21,911 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T23:54:21,913 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T23:54:21,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38331 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731974061909', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:54:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38331 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731974061909 2024-11-18T23:54:21,916 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731974061909 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T23:54:21,916 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:21,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38331 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731974061909" procId is: 4 2024-11-18T23:54:21,917 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731974061909 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T23:54:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:21,932 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731974061909-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a4db809f53e45c451826667d92eb4d74, NAME => 'testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731974061909', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d 2024-11-18T23:54:21,939 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731974061909-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:21,939 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731974061909-pool-0 {}] regionserver.HRegion(1722): Closing a4db809f53e45c451826667d92eb4d74, disabling compactions & flushes 2024-11-18T23:54:21,939 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731974061909-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:21,939 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731974061909-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:21,939 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731974061909-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. after waiting 0 ms 2024-11-18T23:54:21,939 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731974061909-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:21,939 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731974061909-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:21,939 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731974061909-pool-0 {}] regionserver.HRegion(1676): Region close journal for a4db809f53e45c451826667d92eb4d74: Waiting for close lock at 1731974061939Disabling compacts and flushes for region at 1731974061939Disabling writes for close at 1731974061939Writing region close event to WAL at 1731974061939Closed at 1731974061939 2024-11-18T23:54:21,941 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731974061909 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T23:54:21,941 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731974061941"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731974061941"}]},"ts":"1731974061941"} 2024-11-18T23:54:21,944 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T23:54:21,945 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731974061909 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T23:54:21,945 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731974061909","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974061945"}]},"ts":"1731974061945"} 2024-11-18T23:54:21,947 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731974061909, state=ENABLING in hbase:meta 2024-11-18T23:54:21,948 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9c732c6543ff=0} racks are {/default-rack=0} 2024-11-18T23:54:21,948 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T23:54:21,948 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T23:54:21,948 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T23:54:21,948 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T23:54:21,948 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T23:54:21,948 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T23:54:21,948 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T23:54:21,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731974061909, region=a4db809f53e45c451826667d92eb4d74, ASSIGN}] 2024-11-18T23:54:21,950 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731974061909, region=a4db809f53e45c451826667d92eb4d74, ASSIGN 2024-11-18T23:54:21,951 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731974061909, region=a4db809f53e45c451826667d92eb4d74, ASSIGN; state=OFFLINE, location=9c732c6543ff,43617,1731974060867; forceNewPlan=false, retain=false 2024-11-18T23:54:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:22,101 INFO [9c732c6543ff:38331 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T23:54:22,102 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a4db809f53e45c451826667d92eb4d74, regionState=OPENING, regionLocation=9c732c6543ff,43617,1731974060867 2024-11-18T23:54:22,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731974061909, region=a4db809f53e45c451826667d92eb4d74, ASSIGN because future has completed 2024-11-18T23:54:22,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4db809f53e45c451826667d92eb4d74, server=9c732c6543ff,43617,1731974060867}] 2024-11-18T23:54:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:22,260 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:22,261 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a4db809f53e45c451826667d92eb4d74, NAME => 'testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:22,261 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731974061909 a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:22,261 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:22,261 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:22,261 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:22,262 INFO [StoreOpener-a4db809f53e45c451826667d92eb4d74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:22,266 INFO [StoreOpener-a4db809f53e45c451826667d92eb4d74-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4db809f53e45c451826667d92eb4d74 columnFamilyName family 2024-11-18T23:54:22,266 DEBUG [StoreOpener-a4db809f53e45c451826667d92eb4d74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:22,266 INFO [StoreOpener-a4db809f53e45c451826667d92eb4d74-1 {}] regionserver.HStore(327): Store=a4db809f53e45c451826667d92eb4d74/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:22,267 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:22,267 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:22,267 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:22,267 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:22,267 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:22,268 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:22,274 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:54:22,274 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a4db809f53e45c451826667d92eb4d74; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71005891, jitterRate=0.05807022750377655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:54:22,274 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:22,275 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a4db809f53e45c451826667d92eb4d74: Running coprocessor pre-open hook at 1731974062262Writing region info on filesystem at 1731974062262Initializing all the Stores at 1731974062262Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974062262Cleaning up temporary data from old regions at 1731974062267 (+5 ms)Running coprocessor post-open hooks at 1731974062275 (+8 ms)Region opened successfully at 1731974062275 2024-11-18T23:54:22,276 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74., pid=6, masterSystemTime=1731974062258 2024-11-18T23:54:22,278 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:22,279 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:22,279 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a4db809f53e45c451826667d92eb4d74, regionState=OPEN, openSeqNum=2, regionLocation=9c732c6543ff,43617,1731974060867 2024-11-18T23:54:22,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4db809f53e45c451826667d92eb4d74, server=9c732c6543ff,43617,1731974060867 because future has completed 2024-11-18T23:54:22,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T23:54:22,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a4db809f53e45c451826667d92eb4d74, server=9c732c6543ff,43617,1731974060867 in 176 msec 2024-11-18T23:54:22,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T23:54:22,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731974061909, region=a4db809f53e45c451826667d92eb4d74, ASSIGN in 335 msec 2024-11-18T23:54:22,286 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731974061909 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T23:54:22,287 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731974061909","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731974062286"}]},"ts":"1731974062286"} 2024-11-18T23:54:22,289 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731974061909, state=ENABLED in hbase:meta 2024-11-18T23:54:22,289 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731974061909 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T23:54:22,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731974061909 in 375 msec 2024-11-18T23:54:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:54:22,545 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731974061909 get assigned. Timeout = 60000ms 2024-11-18T23:54:22,545 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731974061909 completed 2024-11-18T23:54:22,546 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T23:54:22,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731974061909 assigned to meta. Checking AM states. 2024-11-18T23:54:22,549 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T23:54:22,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731974061909 assigned. 2024-11-18T23:54:22,552 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731974061909', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74., hostname=9c732c6543ff,43617,1731974060867, seqNum=2] 2024-11-18T23:54:22,556 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4db809f53e45c451826667d92eb4d74 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-18T23:54:22,567 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6269e269242f4e21acba12cb5cb42fab 2024-11-18T23:54:22,580 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/.tmp/family/6269e269242f4e21acba12cb5cb42fab is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-18T23:54:22,581 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/.tmp/family/6269e269242f4e21acba12cb5cb42fab 2024-11-18T23:54:22,583 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/.tmp/family/6269e269242f4e21acba12cb5cb42fab as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/family/6269e269242f4e21acba12cb5cb42fab 2024-11-18T23:54:22,584 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/family/6269e269242f4e21acba12cb5cb42fab, delay=949 ms 2024-11-18T23:54:22,585 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/family/6269e269242f4e21acba12cb5cb42fab, entries=2, sequenceid=6, filesize=4.9 K 2024-11-18T23:54:22,586 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for a4db809f53e45c451826667d92eb4d74 in 30ms, sequenceid=6, compaction requested=false 2024-11-18T23:54:22,586 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731974061909' 2024-11-18T23:54:22,587 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a4db809f53e45c451826667d92eb4d74: 2024-11-18T23:54:23,087 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:54:23,535 DEBUG [hfile-prefetch-1731974060217 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6269e269242f4e21acba12cb5cb42fab_0. 2024-11-18T23:54:23,535 DEBUG [hfile-prefetch-1731974060217 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6269e269242f4e21acba12cb5cb42fab_117. 2024-11-18T23:54:23,536 DEBUG [hfile-prefetch-1731974060217 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/family/6269e269242f4e21acba12cb5cb42fab, with total blocks 2, and data blocks 1 2024-11-18T23:54:23,536 DEBUG [hfile-prefetch-1731974060217 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-18T23:54:23,536 DEBUG [hfile-prefetch-1731974060217 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/family/6269e269242f4e21acba12cb5cb42fab, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-18T23:54:23,536 DEBUG [hfile-prefetch-1731974060217 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/family/6269e269242f4e21acba12cb5cb42fab. Will sleep 100ms and try the verification again. 2024-11-18T23:54:23,545 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T23:54:23,545 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T23:54:23,545 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T23:54:23,545 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T23:54:23,549 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T23:54:23,549 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731974063545 2024-11-18T23:54:23,549 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T23:54:23,587 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;9c732c6543ff:43617,5,FailOnTimeoutGroup] 2024-11-18T23:54:23,588 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,43617,1731974060867' ***** 2024-11-18T23:54:23,588 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-18T23:54:23,588 INFO [RS:1;9c732c6543ff:43617 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T23:54:23,588 INFO [RS:1;9c732c6543ff:43617 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.persistence 2024-11-18T23:54:23,588 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T23:54:23,588 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T23:54:23,588 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T23:54:23,588 INFO [RS:1;9c732c6543ff:43617 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T23:54:23,588 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T23:54:23,599 DEBUG [RS:1;9c732c6543ff:43617 {}] bucket.BucketCache(1487): Thread RS:1;9c732c6543ff:43617 started persisting bucket cache to file 2024-11-18T23:54:23,599 DEBUG [RS:1;9c732c6543ff:43617 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T23:54:23,599 DEBUG [RS:1;9c732c6543ff:43617 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T23:54:23,606 DEBUG [RS:1;9c732c6543ff:43617 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-18T23:54:23,606 DEBUG [RS:1;9c732c6543ff:43617 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731974063599 2024-11-18T23:54:23,606 DEBUG [RS:1;9c732c6543ff:43617 {}] bucket.BucketCache(1509): Thread RS:1;9c732c6543ff:43617 finished persisting bucket cache to file, renaming 2024-11-18T23:54:23,606 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T23:54:23,606 INFO [RS:1;9c732c6543ff:43617 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T23:54:23,606 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T23:54:23,606 INFO [RS:1;9c732c6543ff:43617 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T23:54:23,606 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(3091): Received CLOSE for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:23,607 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,43617,1731974060867 2024-11-18T23:54:23,607 INFO [RS:1;9c732c6543ff:43617 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:54:23,607 INFO [RS:1;9c732c6543ff:43617 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9c732c6543ff:43617. 2024-11-18T23:54:23,607 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T23:54:23,607 DEBUG [RS:1;9c732c6543ff:43617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:23,607 DEBUG [RS:1;9c732c6543ff:43617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:23,607 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a4db809f53e45c451826667d92eb4d74, disabling compactions & flushes 2024-11-18T23:54:23,607 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:23,607 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:23,607 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T23:54:23,607 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T23:54:23,607 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. after waiting 0 ms 2024-11-18T23:54:23,607 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T23:54:23,607 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:23,607 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T23:54:23,608 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-18T23:54:23,608 DEBUG [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(1325): Online Regions={a4db809f53e45c451826667d92eb4d74=testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74., 1588230740=hbase:meta,,1.1588230740} 2024-11-18T23:54:23,608 DEBUG [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:23,608 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T23:54:23,608 DEBUG [StoreFileCloser-a4db809f53e45c451826667d92eb4d74-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/family/6269e269242f4e21acba12cb5cb42fab 2024-11-18T23:54:23,608 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T23:54:23,608 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T23:54:23,608 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T23:54:23,608 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T23:54:23,608 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T23:54:23,608 WARN [hfile-prefetch-1731974060217 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/family/6269e269242f4e21acba12cb5cb42fab, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-18T23:54:23,608 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-18T23:54:23,609 DEBUG [hfile-prefetch-1731974060217 {}] hfile.PrefetchExecutor(121): Prefetch completed for 6269e269242f4e21acba12cb5cb42fab 2024-11-18T23:54:23,613 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-18T23:54:23,613 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:23,613 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a4db809f53e45c451826667d92eb4d74: Waiting for close lock at 1731974063607Running coprocessor pre-close hooks at 1731974063607Disabling compacts and flushes for region at 1731974063607Disabling writes for close at 1731974063607Writing region close event to WAL at 1731974063608 (+1 ms)Running coprocessor post-close hooks at 1731974063613 (+5 ms)Closed at 1731974063613 2024-11-18T23:54:23,613 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:23,620 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8b6c1ec7c46d4841bbb04e50c7ae3418 2024-11-18T23:54:23,634 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/info/8b6c1ec7c46d4841bbb04e50c7ae3418 is 209, key is testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74./info:regioninfo/1731974062279/Put/seqid=0 2024-11-18T23:54:23,635 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/info/8b6c1ec7c46d4841bbb04e50c7ae3418 2024-11-18T23:54:23,644 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 46edc4690ee74129a1f9a8f5c23a449b 2024-11-18T23:54:23,649 INFO [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:23,658 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/ns/46edc4690ee74129a1f9a8f5c23a449b is 43, key is default/ns:d/1731974061787/Put/seqid=0 2024-11-18T23:54:23,659 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/ns/46edc4690ee74129a1f9a8f5c23a449b 2024-11-18T23:54:23,668 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cf553a3ed3cf4b9ba43b651c45558a10 2024-11-18T23:54:23,681 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/table/cf553a3ed3cf4b9ba43b651c45558a10 is 80, key is testBlockEvictionOnGracefulStop1731974061909/table:state/1731974062286/Put/seqid=0 2024-11-18T23:54:23,682 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/table/cf553a3ed3cf4b9ba43b651c45558a10 2024-11-18T23:54:23,684 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/info/8b6c1ec7c46d4841bbb04e50c7ae3418 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/8b6c1ec7c46d4841bbb04e50c7ae3418 2024-11-18T23:54:23,686 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/8b6c1ec7c46d4841bbb04e50c7ae3418, delay=907 ms 2024-11-18T23:54:23,686 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/8b6c1ec7c46d4841bbb04e50c7ae3418, entries=10, sequenceid=11, filesize=7.0 K 2024-11-18T23:54:23,687 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/ns/46edc4690ee74129a1f9a8f5c23a449b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/ns/46edc4690ee74129a1f9a8f5c23a449b 2024-11-18T23:54:23,688 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/ns/46edc4690ee74129a1f9a8f5c23a449b, delay=994 ms 2024-11-18T23:54:23,689 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/ns/46edc4690ee74129a1f9a8f5c23a449b, entries=2, sequenceid=11, filesize=5.0 K 2024-11-18T23:54:23,689 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/table/cf553a3ed3cf4b9ba43b651c45558a10 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/table/cf553a3ed3cf4b9ba43b651c45558a10 2024-11-18T23:54:23,690 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/table/cf553a3ed3cf4b9ba43b651c45558a10, delay=904 ms 2024-11-18T23:54:23,691 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/table/cf553a3ed3cf4b9ba43b651c45558a10, entries=2, sequenceid=11, filesize=5.3 K 2024-11-18T23:54:23,692 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=11, compaction requested=false 2024-11-18T23:54:23,692 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/8b6c1ec7c46d4841bbb04e50c7ae3418 2024-11-18T23:54:23,693 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/ns/46edc4690ee74129a1f9a8f5c23a449b 2024-11-18T23:54:23,694 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/table/cf553a3ed3cf4b9ba43b651c45558a10 2024-11-18T23:54:23,698 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-18T23:54:23,699 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T23:54:23,699 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T23:54:23,699 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731974063608Running coprocessor pre-close hooks at 1731974063608Disabling compacts and flushes for region at 1731974063608Disabling writes for close at 1731974063608Obtaining lock to block concurrent updates at 1731974063608Preparing flush snapshotting stores in 1588230740 at 1731974063608Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731974063609 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731974063609Flushing 1588230740/info: creating writer at 1731974063609Flushing 1588230740/info: appending metadata at 1731974063634 (+25 ms)Flushing 1588230740/info: closing flushed file at 1731974063634Flushing 1588230740/ns: creating writer at 1731974063638 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731974063658 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731974063658Flushing 1588230740/table: creating writer at 1731974063661 (+3 ms)Flushing 1588230740/table: appending metadata at 1731974063681 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731974063681Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a9ce08e: reopening flushed file at 1731974063684 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28619b77: reopening flushed file at 1731974063687 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18efb8ff: reopening flushed file at 1731974063689 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=11, compaction requested=false at 1731974063692 (+3 ms)Writing region close event to WAL at 1731974063694 (+2 ms)Running coprocessor post-close hooks at 1731974063699 (+5 ms)Closed at 1731974063699 2024-11-18T23:54:23,699 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T23:54:23,808 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,43617,1731974060867; all regions closed. 2024-11-18T23:54:23,810 DEBUG [RS:1;9c732c6543ff:43617 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs 2024-11-18T23:54:23,810 INFO [RS:1;9c732c6543ff:43617 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9c732c6543ff%2C43617%2C1731974060867.meta:.meta(num 1731974061740) 2024-11-18T23:54:23,812 DEBUG [RS:1;9c732c6543ff:43617 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs 2024-11-18T23:54:23,812 INFO [RS:1;9c732c6543ff:43617 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9c732c6543ff%2C43617%2C1731974060867:(num 1731974061395) 2024-11-18T23:54:23,812 DEBUG [RS:1;9c732c6543ff:43617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:23,812 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:23,812 INFO [RS:1;9c732c6543ff:43617 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:54:23,812 INFO [RS:1;9c732c6543ff:43617 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T23:54:23,812 INFO [RS:1;9c732c6543ff:43617 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:54:23,812 INFO [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:54:23,813 INFO [RS:1;9c732c6543ff:43617 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43617 2024-11-18T23:54:23,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,43617,1731974060867 2024-11-18T23:54:23,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:54:23,817 INFO [RS:1;9c732c6543ff:43617 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:54:23,819 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,43617,1731974060867] 2024-11-18T23:54:23,820 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,43617,1731974060867 already deleted, retry=false 2024-11-18T23:54:23,820 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9c732c6543ff,43617,1731974060867 on 9c732c6543ff,38331,1731974060785 2024-11-18T23:54:23,825 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9c732c6543ff,43617,1731974060867, splitWal=true, meta=true 2024-11-18T23:54:23,827 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9c732c6543ff,43617,1731974060867, splitWal=true, meta=true 2024-11-18T23:54:23,828 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 9c732c6543ff,43617,1731974060867 (carryingMeta=true) 9c732c6543ff,43617,1731974060867/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@bcc63bc[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-18T23:54:23,829 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 9c732c6543ff,43617,1731974060867, splitWal=true, meta=true, isMeta: true 2024-11-18T23:54:23,829 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,43617,1731974060867-splitting 2024-11-18T23:54:23,830 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,43617,1731974060867-splitting dir is empty, no logs to split. 2024-11-18T23:54:23,830 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9c732c6543ff,43617,1731974060867 WAL count=0, meta=true 2024-11-18T23:54:23,832 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,43617,1731974060867-splitting dir is empty, no logs to split. 2024-11-18T23:54:23,832 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9c732c6543ff,43617,1731974060867 WAL count=0, meta=true 2024-11-18T23:54:23,832 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 9c732c6543ff,43617,1731974060867 WAL splitting is done? wals=0, meta=true 2024-11-18T23:54:23,833 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T23:54:23,834 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:54:23,835 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T23:54:23,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:23,919 INFO [RS:1;9c732c6543ff:43617 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:54:23,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43617-0x1010845cf8b0002, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:23,919 INFO [RS:1;9c732c6543ff:43617 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,43617,1731974060867; zookeeper connection closed. 2024-11-18T23:54:23,919 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2493dc7c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2493dc7c 2024-11-18T23:54:23,985 DEBUG [9c732c6543ff:38331 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T23:54:23,986 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9c732c6543ff,38947,1731974060834 2024-11-18T23:54:23,987 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,38947,1731974060834, state=OPENING 2024-11-18T23:54:23,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:23,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:23,989 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:54:23,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:23,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:23,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,38947,1731974060834}] 2024-11-18T23:54:24,105 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9c732c6543ff:0 server-side Connection retries=45 2024-11-18T23:54:24,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:24,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:24,105 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:54:24,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:54:24,106 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:54:24,106 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T23:54:24,106 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:54:24,107 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35483 2024-11-18T23:54:24,108 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35483 connecting to ZooKeeper ensemble=127.0.0.1:52910 2024-11-18T23:54:24,108 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:24,108 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:24,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354830x0, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:54:24,112 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35483-0x1010845cf8b0003 connected 2024-11-18T23:54:24,112 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35483-0x1010845cf8b0003, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-18T23:54:24,112 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-18T23:54:24,113 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T23:54:24,114 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.cache 2024-11-18T23:54:24,114 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T23:54:24,114 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T23:54:24,114 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T23:54:24,114 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T23:54:24,114 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T23:54:24,115 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T23:54:24,115 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-18T23:54:24,115 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35483-0x1010845cf8b0003, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T23:54:24,117 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-18T23:54:24,117 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35483-0x1010845cf8b0003, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:54:24,121 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35483 2024-11-18T23:54:24,121 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35483 2024-11-18T23:54:24,122 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35483 2024-11-18T23:54:24,122 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35483 2024-11-18T23:54:24,122 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35483 2024-11-18T23:54:24,124 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer(746): ClusterId : 02b9c948-49eb-48f0-922c-12740ac403a4 2024-11-18T23:54:24,124 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T23:54:24,124 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-18T23:54:24,125 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T23:54:24,125 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-18T23:54:24,125 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.persistence is complete. 2024-11-18T23:54:24,126 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T23:54:24,126 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T23:54:24,128 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T23:54:24,129 DEBUG [RS:2;9c732c6543ff:35483 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26332baf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9c732c6543ff/172.17.0.2:0 2024-11-18T23:54:24,140 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:54:24,142 DEBUG [RS:2;9c732c6543ff:35483 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9c732c6543ff:35483 2024-11-18T23:54:24,142 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T23:54:24,142 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T23:54:24,142 DEBUG [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T23:54:24,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48513, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:54:24,143 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer(2659): reportForDuty to master=9c732c6543ff,38331,1731974060785 with port=35483, startcode=1731974064105 2024-11-18T23:54:24,143 DEBUG [RS:2;9c732c6543ff:35483 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T23:54:24,144 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T23:54:24,144 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:24,145 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T23:54:24,145 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33861, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T23:54:24,146 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9c732c6543ff%2C38947%2C1731974060834.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,38947,1731974060834, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs, maxLogs=55 2024-11-18T23:54:24,146 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38331 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9c732c6543ff,35483,1731974064105 2024-11-18T23:54:24,146 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38331 {}] master.ServerManager(517): Registering regionserver=9c732c6543ff,35483,1731974064105 2024-11-18T23:54:24,147 DEBUG [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d 2024-11-18T23:54:24,147 DEBUG [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T23:54:24,147 DEBUG [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T23:54:24,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:54:24,150 DEBUG [RS:2;9c732c6543ff:35483 {}] zookeeper.ZKUtil(111): regionserver:35483-0x1010845cf8b0003, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9c732c6543ff,35483,1731974064105 2024-11-18T23:54:24,150 WARN [RS:2;9c732c6543ff:35483 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:54:24,150 INFO [RS:2;9c732c6543ff:35483 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:54:24,150 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9c732c6543ff,35483,1731974064105] 2024-11-18T23:54:24,150 DEBUG [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,35483,1731974064105 2024-11-18T23:54:24,156 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T23:54:24,159 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T23:54:24,159 INFO [RS:2;9c732c6543ff:35483 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:54:24,159 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:24,160 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T23:54:24,160 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T23:54:24,160 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:24,160 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9c732c6543ff:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9c732c6543ff:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:24,161 DEBUG [RS:2;9c732c6543ff:35483 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9c732c6543ff:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:54:24,162 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:24,162 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:24,162 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:24,162 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:24,162 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:24,162 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,35483,1731974064105-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:54:24,166 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,38947,1731974060834/9c732c6543ff%2C38947%2C1731974060834.meta.1731974064146.meta 2024-11-18T23:54:24,166 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:24,166 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:24,167 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T23:54:24,167 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T23:54:24,167 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T23:54:24,167 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T23:54:24,167 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:24,167 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T23:54:24,167 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T23:54:24,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T23:54:24,169 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T23:54:24,169 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:24,170 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/8b6c1ec7c46d4841bbb04e50c7ae3418, delay=978 ms 2024-11-18T23:54:24,171 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/8b6c1ec7c46d4841bbb04e50c7ae3418 2024-11-18T23:54:24,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:24,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T23:54:24,172 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T23:54:24,172 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:24,174 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/ns/46edc4690ee74129a1f9a8f5c23a449b, delay=902 ms 2024-11-18T23:54:24,176 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/ns/46edc4690ee74129a1f9a8f5c23a449b 2024-11-18T23:54:24,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:24,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T23:54:24,177 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T23:54:24,177 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:24,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:24,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T23:54:24,178 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T23:54:24,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:24,179 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T23:54:24,179 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(168): Chore ScheduledChore name=9c732c6543ff,35483,1731974064105-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:24,180 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:24,180 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.Replication(171): 9c732c6543ff,35483,1731974064105 started 2024-11-18T23:54:24,180 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/table/cf553a3ed3cf4b9ba43b651c45558a10, delay=907 ms 2024-11-18T23:54:24,181 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/table/cf553a3ed3cf4b9ba43b651c45558a10 2024-11-18T23:54:24,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:54:24,181 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T23:54:24,182 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740 2024-11-18T23:54:24,182 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740 2024-11-18T23:54:24,183 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T23:54:24,183 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T23:54:24,183 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:54:24,185 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T23:54:24,185 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66910975, jitterRate=-0.002948775887489319}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:54:24,185 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T23:54:24,187 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731974064167Writing region info on filesystem at 1731974064167Initializing all the Stores at 1731974064168 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974064168Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974064168Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974064168Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731974064168Cleaning up temporary data from old regions at 1731974064183 (+15 ms)Running coprocessor post-open hooks at 1731974064185 (+2 ms)Region opened successfully at 1731974064186 (+1 ms) 2024-11-18T23:54:24,187 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731974064140 2024-11-18T23:54:24,190 DEBUG [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T23:54:24,191 INFO [RS_OPEN_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T23:54:24,191 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=9c732c6543ff,38947,1731974060834 2024-11-18T23:54:24,192 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9c732c6543ff,38947,1731974060834, state=OPEN 2024-11-18T23:54:24,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:24,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35483-0x1010845cf8b0003, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:24,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:54:24,194 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:24,194 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:24,194 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:54:24,194 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9c732c6543ff,38947,1731974060834 2024-11-18T23:54:24,196 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:54:24,196 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer(1482): Serving as 9c732c6543ff,35483,1731974064105, RpcServer on 9c732c6543ff/172.17.0.2:35483, sessionid=0x1010845cf8b0003 2024-11-18T23:54:24,196 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T23:54:24,196 DEBUG [RS:2;9c732c6543ff:35483 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9c732c6543ff,35483,1731974064105 2024-11-18T23:54:24,196 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,35483,1731974064105' 2024-11-18T23:54:24,196 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T23:54:24,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-18T23:54:24,197 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T23:54:24,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9c732c6543ff,38947,1731974060834 in 205 msec 2024-11-18T23:54:24,197 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T23:54:24,198 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T23:54:24,198 DEBUG [RS:2;9c732c6543ff:35483 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9c732c6543ff,35483,1731974064105 2024-11-18T23:54:24,198 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9c732c6543ff,35483,1731974064105' 2024-11-18T23:54:24,198 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T23:54:24,198 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T23:54:24,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T23:54:24,199 DEBUG [RS:2;9c732c6543ff:35483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T23:54:24,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 364 msec 2024-11-18T23:54:24,199 INFO [RS:2;9c732c6543ff:35483 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T23:54:24,199 INFO [RS:2;9c732c6543ff:35483 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T23:54:24,199 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 9c732c6543ff,43617,1731974060867 had 2 regions 2024-11-18T23:54:24,200 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9c732c6543ff,43617,1731974060867, splitWal=true, meta=true, isMeta: false 2024-11-18T23:54:24,201 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,43617,1731974060867-splitting dir is empty, no logs to split. 2024-11-18T23:54:24,201 INFO [PEWorker-3 {}] master.SplitWALManager(105): 9c732c6543ff,43617,1731974060867 WAL count=0, meta=false 2024-11-18T23:54:24,202 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,43617,1731974060867-splitting dir is empty, no logs to split. 2024-11-18T23:54:24,202 INFO [PEWorker-3 {}] master.SplitWALManager(105): 9c732c6543ff,43617,1731974060867 WAL count=0, meta=false 2024-11-18T23:54:24,202 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 9c732c6543ff,43617,1731974060867 WAL splitting is done? wals=0, meta=false 2024-11-18T23:54:24,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731974061909, region=a4db809f53e45c451826667d92eb4d74, ASSIGN}] 2024-11-18T23:54:24,204 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731974061909, region=a4db809f53e45c451826667d92eb4d74, ASSIGN 2024-11-18T23:54:24,205 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731974061909, region=a4db809f53e45c451826667d92eb4d74, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T23:54:24,300 INFO [RS:2;9c732c6543ff:35483 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:54:24,300 INFO [RS:2;9c732c6543ff:35483 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9c732c6543ff%2C35483%2C1731974064105, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,35483,1731974064105, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs, maxLogs=55 2024-11-18T23:54:24,320 INFO [RS:2;9c732c6543ff:35483 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/WALs/9c732c6543ff,35483,1731974064105/9c732c6543ff%2C35483%2C1731974064105.1731974064301 2024-11-18T23:54:24,321 DEBUG [RS:2;9c732c6543ff:35483 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T23:54:24,356 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(204): Hosts are {9c732c6543ff=0} racks are {/default-rack=0} 2024-11-18T23:54:24,356 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T23:54:24,356 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T23:54:24,356 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T23:54:24,356 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T23:54:24,356 INFO [9c732c6543ff:38331 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T23:54:24,356 INFO [9c732c6543ff:38331 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T23:54:24,356 DEBUG [9c732c6543ff:38331 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T23:54:24,357 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a4db809f53e45c451826667d92eb4d74, regionState=OPENING, regionLocation=9c732c6543ff,38947,1731974060834 2024-11-18T23:54:24,358 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9c732c6543ff:43617 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9c732c6543ff/172.17.0.2:43617 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:54:24,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,43617,1731974060867, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,43617,1731974060867, seqNum=-1, error=java.net.ConnectException: Call to address=9c732c6543ff:43617 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9c732c6543ff/172.17.0.2:43617 2024-11-18T23:54:24,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,43617,1731974060867, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T23:54:24,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,43617,1731974060867, seqNum=-1 from cache 2024-11-18T23:54:24,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 9c732c6543ff:43617 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9c732c6543ff/172.17.0.2:43617 2024-11-18T23:54:24,463 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:54:24,464 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9c732c6543ff,38947,1731974060834, seqNum=-1] 2024-11-18T23:54:24,464 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:54:24,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39245, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:54:24,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731974061909, region=a4db809f53e45c451826667d92eb4d74, ASSIGN because future has completed 2024-11-18T23:54:24,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4db809f53e45c451826667d92eb4d74, server=9c732c6543ff,38947,1731974060834}] 2024-11-18T23:54:24,624 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:24,624 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => a4db809f53e45c451826667d92eb4d74, NAME => 'testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:54:24,625 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731974061909 a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,625 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:54:24,625 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,625 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,626 INFO [StoreOpener-a4db809f53e45c451826667d92eb4d74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,627 INFO [StoreOpener-a4db809f53e45c451826667d92eb4d74-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4db809f53e45c451826667d92eb4d74 columnFamilyName family 2024-11-18T23:54:24,627 DEBUG [StoreOpener-a4db809f53e45c451826667d92eb4d74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:54:24,628 DEBUG [StoreFileOpener-a4db809f53e45c451826667d92eb4d74-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/family/6269e269242f4e21acba12cb5cb42fab, delay=917 ms 2024-11-18T23:54:24,629 DEBUG [StoreOpener-a4db809f53e45c451826667d92eb4d74-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/family/6269e269242f4e21acba12cb5cb42fab 2024-11-18T23:54:24,630 INFO [StoreOpener-a4db809f53e45c451826667d92eb4d74-1 {}] regionserver.HStore(327): Store=a4db809f53e45c451826667d92eb4d74/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:54:24,630 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,630 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,631 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,631 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,631 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,632 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,633 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened a4db809f53e45c451826667d92eb4d74; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63898640, jitterRate=-0.0478360652923584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:54:24,633 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,634 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for a4db809f53e45c451826667d92eb4d74: Running coprocessor pre-open hook at 1731974064625Writing region info on filesystem at 1731974064625Initializing all the Stores at 1731974064625Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731974064626 (+1 ms)Cleaning up temporary data from old regions at 1731974064631 (+5 ms)Running coprocessor post-open hooks at 1731974064633 (+2 ms)Region opened successfully at 1731974064634 (+1 ms) 2024-11-18T23:54:24,635 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74., pid=11, masterSystemTime=1731974064621 2024-11-18T23:54:24,637 DEBUG [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:24,637 INFO [RS_OPEN_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:24,638 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a4db809f53e45c451826667d92eb4d74, regionState=OPEN, openSeqNum=10, regionLocation=9c732c6543ff,38947,1731974060834 2024-11-18T23:54:24,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4db809f53e45c451826667d92eb4d74, server=9c732c6543ff,38947,1731974060834 because future has completed 2024-11-18T23:54:24,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-18T23:54:24,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure a4db809f53e45c451826667d92eb4d74, server=9c732c6543ff,38947,1731974060834 in 173 msec 2024-11-18T23:54:24,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-18T23:54:24,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731974061909, region=a4db809f53e45c451826667d92eb4d74, ASSIGN in 440 msec 2024-11-18T23:54:24,645 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 9c732c6543ff,43617,1731974060867 after splitting done 2024-11-18T23:54:24,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 9c732c6543ff,43617,1731974060867, splitWal=true, meta=true in 825 msec 2024-11-18T23:54:24,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T23:54:24,696 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:54:24,696 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:24,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:24,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:24,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:54:24,697 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T23:54:24,697 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1324664382, stopped=false 2024-11-18T23:54:24,697 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9c732c6543ff,38331,1731974060785 2024-11-18T23:54:24,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:24,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:24,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35483-0x1010845cf8b0003, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:54:24,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:24,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35483-0x1010845cf8b0003, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:24,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:24,699 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T23:54:24,699 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:54:24,699 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:24,700 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:24,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:24,700 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,38947,1731974060834' ***** 2024-11-18T23:54:24,700 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T23:54:24,700 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9c732c6543ff,35483,1731974064105' ***** 2024-11-18T23:54:24,700 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T23:54:24,700 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:24,700 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35483-0x1010845cf8b0003, quorum=127.0.0.1:52910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:54:24,700 INFO [RS:0;9c732c6543ff:38947 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T23:54:24,700 INFO [RS:2;9c732c6543ff:35483 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T23:54:24,701 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T23:54:24,701 INFO [RS:2;9c732c6543ff:35483 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.persistence 2024-11-18T23:54:24,701 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T23:54:24,701 INFO [RS:0;9c732c6543ff:38947 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a60eb205-5d50-4bc0-2611-4f61cdd4b227/bucket.persistence 2024-11-18T23:54:24,701 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T23:54:24,701 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T23:54:24,701 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T23:54:24,701 INFO [RS:0;9c732c6543ff:38947 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T23:54:24,701 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T23:54:24,701 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T23:54:24,702 INFO [RS:2;9c732c6543ff:35483 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T23:54:24,702 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T23:54:24,711 DEBUG [RS:0;9c732c6543ff:38947 {}] bucket.BucketCache(1487): Thread RS:0;9c732c6543ff:38947 started persisting bucket cache to file 2024-11-18T23:54:24,712 DEBUG [RS:0;9c732c6543ff:38947 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T23:54:24,712 DEBUG [RS:0;9c732c6543ff:38947 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T23:54:24,712 DEBUG [RS:2;9c732c6543ff:35483 {}] bucket.BucketCache(1487): Thread RS:2;9c732c6543ff:35483 started persisting bucket cache to file 2024-11-18T23:54:24,712 DEBUG [RS:2;9c732c6543ff:35483 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T23:54:24,712 DEBUG [RS:2;9c732c6543ff:35483 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T23:54:24,716 DEBUG [RS:0;9c732c6543ff:38947 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T23:54:24,716 DEBUG [RS:0;9c732c6543ff:38947 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731974064711 2024-11-18T23:54:24,716 DEBUG [RS:0;9c732c6543ff:38947 {}] bucket.BucketCache(1509): Thread RS:0;9c732c6543ff:38947 finished persisting bucket cache to file, renaming 2024-11-18T23:54:24,716 DEBUG [RS:2;9c732c6543ff:35483 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-18T23:54:24,716 DEBUG [RS:2;9c732c6543ff:35483 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731974064712 2024-11-18T23:54:24,716 DEBUG [RS:2;9c732c6543ff:35483 {}] bucket.BucketCache(1509): Thread RS:2;9c732c6543ff:35483 finished persisting bucket cache to file, renaming 2024-11-18T23:54:24,716 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T23:54:24,716 INFO [RS:0;9c732c6543ff:38947 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T23:54:24,716 INFO [RS:0;9c732c6543ff:38947 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T23:54:24,716 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T23:54:24,716 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(3091): Received CLOSE for a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,716 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,38947,1731974060834 2024-11-18T23:54:24,716 INFO [RS:0;9c732c6543ff:38947 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:54:24,717 INFO [RS:0;9c732c6543ff:38947 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9c732c6543ff:38947. 2024-11-18T23:54:24,717 DEBUG [RS:0;9c732c6543ff:38947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:24,717 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T23:54:24,717 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T23:54:24,717 DEBUG [RS:0;9c732c6543ff:38947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:24,717 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a4db809f53e45c451826667d92eb4d74, disabling compactions & flushes 2024-11-18T23:54:24,717 INFO [RS:2;9c732c6543ff:35483 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T23:54:24,717 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T23:54:24,717 INFO [RS:2;9c732c6543ff:35483 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T23:54:24,717 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:24,717 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T23:54:24,717 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer(959): stopping server 9c732c6543ff,35483,1731974064105 2024-11-18T23:54:24,717 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:24,717 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T23:54:24,717 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T23:54:24,717 INFO [RS:2;9c732c6543ff:35483 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:54:24,717 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. after waiting 0 ms 2024-11-18T23:54:24,717 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T23:54:24,717 INFO [RS:2;9c732c6543ff:35483 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9c732c6543ff:35483. 2024-11-18T23:54:24,717 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:24,717 DEBUG [RS:2;9c732c6543ff:35483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:54:24,717 DEBUG [RS:2;9c732c6543ff:35483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:24,717 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-18T23:54:24,717 DEBUG [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(1325): Online Regions={a4db809f53e45c451826667d92eb4d74=testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74., 1588230740=hbase:meta,,1.1588230740} 2024-11-18T23:54:24,717 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,35483,1731974064105; all regions closed. 2024-11-18T23:54:24,717 DEBUG [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, a4db809f53e45c451826667d92eb4d74 2024-11-18T23:54:24,718 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T23:54:24,718 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T23:54:24,718 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T23:54:24,718 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T23:54:24,718 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T23:54:24,718 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T23:54:24,718 DEBUG [StoreFileCloser-a4db809f53e45c451826667d92eb4d74-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/family/6269e269242f4e21acba12cb5cb42fab 2024-11-18T23:54:24,718 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-18T23:54:24,719 DEBUG [RS:2;9c732c6543ff:35483 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs 2024-11-18T23:54:24,720 INFO [RS:2;9c732c6543ff:35483 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9c732c6543ff%2C35483%2C1731974064105:(num 1731974064301) 2024-11-18T23:54:24,720 DEBUG [RS:2;9c732c6543ff:35483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:24,720 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:24,720 INFO [RS:2;9c732c6543ff:35483 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:54:24,720 INFO [RS:2;9c732c6543ff:35483 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T23:54:24,720 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T23:54:24,720 INFO [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:54:24,720 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T23:54:24,720 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T23:54:24,720 INFO [RS:2;9c732c6543ff:35483 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:54:24,720 INFO [RS:2;9c732c6543ff:35483 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35483 2024-11-18T23:54:24,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:54:24,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35483-0x1010845cf8b0003, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,35483,1731974064105 2024-11-18T23:54:24,723 INFO [RS:2;9c732c6543ff:35483 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:54:24,723 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/default/testBlockEvictionOnGracefulStop1731974061909/a4db809f53e45c451826667d92eb4d74/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-18T23:54:24,723 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,35483,1731974064105] 2024-11-18T23:54:24,724 INFO [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:24,724 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a4db809f53e45c451826667d92eb4d74: Waiting for close lock at 1731974064717Running coprocessor pre-close hooks at 1731974064717Disabling compacts and flushes for region at 1731974064717Disabling writes for close at 1731974064717Writing region close event to WAL at 1731974064718 (+1 ms)Running coprocessor post-close hooks at 1731974064724 (+6 ms)Closed at 1731974064724 2024-11-18T23:54:24,724 DEBUG [RS_CLOSE_REGION-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74. 2024-11-18T23:54:24,725 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,35483,1731974064105 already deleted, retry=false 2024-11-18T23:54:24,725 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,35483,1731974064105 expired; onlineServers=1 2024-11-18T23:54:24,730 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7cc002090b664dd7a5c8d19ba95005f2 2024-11-18T23:54:24,744 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/info/7cc002090b664dd7a5c8d19ba95005f2 is 209, key is testBlockEvictionOnGracefulStop1731974061909,,1731974061913.a4db809f53e45c451826667d92eb4d74./info:regioninfo/1731974064638/Put/seqid=0 2024-11-18T23:54:24,745 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/info/7cc002090b664dd7a5c8d19ba95005f2 2024-11-18T23:54:24,747 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/.tmp/info/7cc002090b664dd7a5c8d19ba95005f2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/7cc002090b664dd7a5c8d19ba95005f2 2024-11-18T23:54:24,748 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/7cc002090b664dd7a5c8d19ba95005f2, delay=993 ms 2024-11-18T23:54:24,749 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/7cc002090b664dd7a5c8d19ba95005f2, entries=8, sequenceid=19, filesize=6.6 K 2024-11-18T23:54:24,750 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false 2024-11-18T23:54:24,751 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/8b6c1ec7c46d4841bbb04e50c7ae3418 2024-11-18T23:54:24,751 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/info/7cc002090b664dd7a5c8d19ba95005f2 2024-11-18T23:54:24,751 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/ns/46edc4690ee74129a1f9a8f5c23a449b 2024-11-18T23:54:24,752 INFO [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:24,752 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/table/cf553a3ed3cf4b9ba43b651c45558a10 2024-11-18T23:54:24,756 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-18T23:54:24,756 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T23:54:24,756 INFO [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T23:54:24,756 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731974064717Running coprocessor pre-close hooks at 1731974064718 (+1 ms)Disabling compacts and flushes for region at 1731974064718Disabling writes for close at 1731974064718Obtaining lock to block concurrent updates at 1731974064718Preparing flush snapshotting stores in 1588230740 at 1731974064718Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731974064719 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731974064719Flushing 1588230740/info: creating writer at 1731974064719Flushing 1588230740/info: appending metadata at 1731974064744 (+25 ms)Flushing 1588230740/info: closing flushed file at 1731974064744Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6830b902: reopening flushed file at 1731974064747 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false at 1731974064750 (+3 ms)Writing region close event to WAL at 1731974064752 (+2 ms)Running coprocessor post-close hooks at 1731974064756 (+4 ms)Closed at 1731974064756 2024-11-18T23:54:24,757 DEBUG [RS_CLOSE_META-regionserver/9c732c6543ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T23:54:24,763 INFO [regionserver/9c732c6543ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:24,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35483-0x1010845cf8b0003, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:24,824 INFO [RS:2;9c732c6543ff:35483 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:54:24,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35483-0x1010845cf8b0003, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:24,825 INFO [RS:2;9c732c6543ff:35483 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,35483,1731974064105; zookeeper connection closed. 2024-11-18T23:54:24,825 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@10e59963 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@10e59963 2024-11-18T23:54:24,918 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(976): stopping server 9c732c6543ff,38947,1731974060834; all regions closed. 2024-11-18T23:54:24,920 DEBUG [RS:0;9c732c6543ff:38947 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs 2024-11-18T23:54:24,920 INFO [RS:0;9c732c6543ff:38947 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9c732c6543ff%2C38947%2C1731974060834.meta:.meta(num 1731974064146) 2024-11-18T23:54:24,921 DEBUG [RS:0;9c732c6543ff:38947 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/oldWALs 2024-11-18T23:54:24,921 INFO [RS:0;9c732c6543ff:38947 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9c732c6543ff%2C38947%2C1731974060834:(num 1731974061395) 2024-11-18T23:54:24,921 DEBUG [RS:0;9c732c6543ff:38947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:54:24,921 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:54:24,922 INFO [RS:0;9c732c6543ff:38947 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:54:24,922 INFO [RS:0;9c732c6543ff:38947 {}] hbase.ChoreService(370): Chore service for: regionserver/9c732c6543ff:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T23:54:24,922 INFO [RS:0;9c732c6543ff:38947 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:54:24,922 INFO [regionserver/9c732c6543ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:54:24,922 INFO [RS:0;9c732c6543ff:38947 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38947 2024-11-18T23:54:24,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9c732c6543ff,38947,1731974060834 2024-11-18T23:54:24,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:54:24,924 INFO [RS:0;9c732c6543ff:38947 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:54:24,926 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9c732c6543ff,38947,1731974060834] 2024-11-18T23:54:24,927 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9c732c6543ff,38947,1731974060834 already deleted, retry=false 2024-11-18T23:54:24,928 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9c732c6543ff,38947,1731974060834 expired; onlineServers=0 2024-11-18T23:54:24,928 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9c732c6543ff,38331,1731974060785' ***** 2024-11-18T23:54:24,928 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T23:54:24,928 INFO [M:0;9c732c6543ff:38331 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:54:24,928 INFO [M:0;9c732c6543ff:38331 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:54:24,928 DEBUG [M:0;9c732c6543ff:38331 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T23:54:24,928 DEBUG [M:0;9c732c6543ff:38331 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T23:54:24,928 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T23:54:24,928 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974061123 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.large.0-1731974061123,5,FailOnTimeoutGroup] 2024-11-18T23:54:24,928 DEBUG [master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974061124 {}] cleaner.HFileCleaner(306): Exit Thread[master/9c732c6543ff:0:becomeActiveMaster-HFileCleaner.small.0-1731974061124,5,FailOnTimeoutGroup] 2024-11-18T23:54:24,928 INFO [M:0;9c732c6543ff:38331 {}] hbase.ChoreService(370): Chore service for: master/9c732c6543ff:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T23:54:24,928 INFO [M:0;9c732c6543ff:38331 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:54:24,928 DEBUG [M:0;9c732c6543ff:38331 {}] master.HMaster(1795): Stopping service threads 2024-11-18T23:54:24,929 INFO [M:0;9c732c6543ff:38331 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T23:54:24,929 INFO [M:0;9c732c6543ff:38331 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T23:54:24,929 INFO [M:0;9c732c6543ff:38331 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T23:54:24,929 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T23:54:24,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T23:54:24,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:54:24,930 DEBUG [M:0;9c732c6543ff:38331 {}] zookeeper.ZKUtil(347): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T23:54:24,930 WARN [M:0;9c732c6543ff:38331 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T23:54:24,930 INFO [M:0;9c732c6543ff:38331 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/.lastflushedseqids 2024-11-18T23:54:24,937 INFO [M:0;9c732c6543ff:38331 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T23:54:24,937 INFO [M:0;9c732c6543ff:38331 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T23:54:24,937 DEBUG [M:0;9c732c6543ff:38331 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T23:54:24,937 INFO [M:0;9c732c6543ff:38331 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:24,937 DEBUG [M:0;9c732c6543ff:38331 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:24,937 DEBUG [M:0;9c732c6543ff:38331 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T23:54:24,937 DEBUG [M:0;9c732c6543ff:38331 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:24,937 INFO [M:0;9c732c6543ff:38331 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-18T23:54:24,947 DEBUG [M:0;9c732c6543ff:38331 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7c723e9efe26408a83dfe7201b11481b 2024-11-18T23:54:24,960 DEBUG [M:0;9c732c6543ff:38331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7c723e9efe26408a83dfe7201b11481b is 82, key is hbase:meta,,1/info:regioninfo/1731974064191/Put/seqid=0 2024-11-18T23:54:24,960 INFO [M:0;9c732c6543ff:38331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7c723e9efe26408a83dfe7201b11481b 2024-11-18T23:54:24,968 DEBUG [M:0;9c732c6543ff:38331 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9aa86ab343184ae8afb3b0c6eb2757ea 2024-11-18T23:54:24,982 DEBUG [M:0;9c732c6543ff:38331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9aa86ab343184ae8afb3b0c6eb2757ea is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731974062290/Put/seqid=0 2024-11-18T23:54:24,983 INFO [M:0;9c732c6543ff:38331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9aa86ab343184ae8afb3b0c6eb2757ea 2024-11-18T23:54:24,990 DEBUG [M:0;9c732c6543ff:38331 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d8c9cb46ac4f40e999efbcd7d33d2403 2024-11-18T23:54:25,004 DEBUG [M:0;9c732c6543ff:38331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d8c9cb46ac4f40e999efbcd7d33d2403 is 69, key is 9c732c6543ff,35483,1731974064105/rs:state/1731974064146/Put/seqid=0 2024-11-18T23:54:25,004 INFO [M:0;9c732c6543ff:38331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d8c9cb46ac4f40e999efbcd7d33d2403 2024-11-18T23:54:25,006 INFO [M:0;9c732c6543ff:38331 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d8c9cb46ac4f40e999efbcd7d33d2403 2024-11-18T23:54:25,006 DEBUG [M:0;9c732c6543ff:38331 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7c723e9efe26408a83dfe7201b11481b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7c723e9efe26408a83dfe7201b11481b 2024-11-18T23:54:25,008 INFO [M:0;9c732c6543ff:38331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7c723e9efe26408a83dfe7201b11481b, entries=14, sequenceid=102, filesize=5.9 K 2024-11-18T23:54:25,008 DEBUG [M:0;9c732c6543ff:38331 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9aa86ab343184ae8afb3b0c6eb2757ea as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9aa86ab343184ae8afb3b0c6eb2757ea 2024-11-18T23:54:25,009 INFO [M:0;9c732c6543ff:38331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9aa86ab343184ae8afb3b0c6eb2757ea, entries=11, sequenceid=102, filesize=6.5 K 2024-11-18T23:54:25,010 DEBUG [M:0;9c732c6543ff:38331 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d8c9cb46ac4f40e999efbcd7d33d2403 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d8c9cb46ac4f40e999efbcd7d33d2403 2024-11-18T23:54:25,011 INFO [M:0;9c732c6543ff:38331 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d8c9cb46ac4f40e999efbcd7d33d2403 2024-11-18T23:54:25,011 INFO [M:0;9c732c6543ff:38331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/54fd8432-ab23-b511-2e1f-c47f8fd5aa7e/f91f1ebb-d9ad-2b84-a941-1db461775b6d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d8c9cb46ac4f40e999efbcd7d33d2403, entries=3, sequenceid=102, filesize=5.3 K 2024-11-18T23:54:25,012 INFO [M:0;9c732c6543ff:38331 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 75ms, sequenceid=102, compaction requested=false 2024-11-18T23:54:25,013 INFO [M:0;9c732c6543ff:38331 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:54:25,014 DEBUG [M:0;9c732c6543ff:38331 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731974064937Disabling compacts and flushes for region at 1731974064937Disabling writes for close at 1731974064937Obtaining lock to block concurrent updates at 1731974064937Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731974064937Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1731974064938 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731974064938Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731974064938Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731974064959 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731974064959Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731974064962 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731974064982 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731974064982Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731974064984 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731974065003 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731974065003Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25e8cbf9: reopening flushed file at 1731974065006 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d760d9: reopening flushed file at 1731974065008 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@de29987: reopening flushed file at 1731974065009 (+1 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 75ms, sequenceid=102, compaction requested=false at 1731974065012 (+3 ms)Writing region close event to WAL at 1731974065013 (+1 ms)Closed at 1731974065013 2024-11-18T23:54:25,015 INFO [M:0;9c732c6543ff:38331 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T23:54:25,015 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:54:25,015 INFO [M:0;9c732c6543ff:38331 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38331 2024-11-18T23:54:25,015 INFO [M:0;9c732c6543ff:38331 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:54:25,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:25,026 INFO [RS:0;9c732c6543ff:38947 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:54:25,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38947-0x1010845cf8b0001, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:25,026 INFO [RS:0;9c732c6543ff:38947 {}] regionserver.HRegionServer(1031): Exiting; stopping=9c732c6543ff,38947,1731974060834; zookeeper connection closed. 2024-11-18T23:54:25,027 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@57c02e33 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@57c02e33 2024-11-18T23:54:25,027 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-18T23:54:25,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:25,117 INFO [M:0;9c732c6543ff:38331 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:54:25,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38331-0x1010845cf8b0000, quorum=127.0.0.1:52910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:54:25,120 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T23:54:25,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T23:54:25,121 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:54:25,128 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 50) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=98 (was 106), ProcessCount=11 (was 11), AvailableMemoryMB=4795 (was 4890)